Compare commits
23 Commits
dec05821bf
...
v0.12.0
| Author | SHA1 | Date | |
|---|---|---|---|
| 74411e3f54 | |||
| 4358a7cdce | |||
| dcea5261ab | |||
| 97141c7a9b | |||
| 2e2d36024e | |||
| 4d2cddf801 | |||
| 21a35a5205 | |||
| 435d1ade07 | |||
| 819ac24aed | |||
| 2c472449e8 | |||
| 5d7f604a2c | |||
| 8041f99782 | |||
| c417c892d9 | |||
| 481215c5db | |||
| 5d94d3f32d | |||
| f8f1b8d4a5 | |||
| 06d5973746 | |||
| 8045b27173 | |||
| 985468c1b9 | |||
| 6a0b30b7c7 | |||
| 86ce4eb68c | |||
| 33541a71fc | |||
| b7277e0c02 |
@@ -1,7 +1,8 @@
|
|||||||
.git
|
.git
|
||||||
.gitignore
|
.gitignore
|
||||||
**/*.md
|
**/*.md
|
||||||
|
!docs/*.md
|
||||||
|
!docs/**/*.md
|
||||||
dist/
|
dist/
|
||||||
tmp/
|
tmp/
|
||||||
.DS_Store
|
.DS_Store
|
||||||
|
|
||||||
|
|||||||
@@ -29,8 +29,10 @@ current working directory.
|
|||||||
- [Operations guide](docs/operations.md)
|
- [Operations guide](docs/operations.md)
|
||||||
- [Troubleshooting guide](docs/troubleshooting.md)
|
- [Troubleshooting guide](docs/troubleshooting.md)
|
||||||
- [Example configs](examples/)
|
- [Example configs](examples/)
|
||||||
|
- [Go consumer guide](docs/consumers/api.md)
|
||||||
- [Event wire contract](docs/integrations/events.md)
|
- [Event wire contract](docs/integrations/events.md)
|
||||||
- [Postgres table contract](docs/integrations/postgres.md)
|
- [Postgres table contract](docs/integrations/postgres.md)
|
||||||
|
- [Feedkit integration notes](docs/integrations/feedkit.md)
|
||||||
- [NWS integration notes](docs/integrations/nws.md)
|
- [NWS integration notes](docs/integrations/nws.md)
|
||||||
- [SPC integration notes](docs/integrations/spc.md)
|
- [SPC integration notes](docs/integrations/spc.md)
|
||||||
- [Open-Meteo integration notes](docs/integrations/openmeteo.md)
|
- [Open-Meteo integration notes](docs/integrations/openmeteo.md)
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ import (
|
|||||||
|
|
||||||
wfnormalizers "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers"
|
wfnormalizers "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers"
|
||||||
wfsources "gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources"
|
wfsources "gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||||
)
|
)
|
||||||
|
|
||||||
type testInput struct {
|
type testInput struct {
|
||||||
@@ -36,10 +37,10 @@ type testKindsSource struct {
|
|||||||
func (s testKindsSource) Kinds() []fkevent.Kind { return s.kinds }
|
func (s testKindsSource) Kinds() []fkevent.Kind { return s.kinds }
|
||||||
|
|
||||||
func TestValidateSourceExpectedKindsSubsetAllowed(t *testing.T) {
|
func TestValidateSourceExpectedKindsSubsetAllowed(t *testing.T) {
|
||||||
sc := config.SourceConfig{Kinds: []string{"observation"}}
|
sc := config.SourceConfig{Kinds: []string{standards.KindObservation}}
|
||||||
in := testKindsSource{
|
in := testKindsSource{
|
||||||
testInput: testInput{name: "test"},
|
testInput: testInput{name: "test"},
|
||||||
kinds: []fkevent.Kind{"observation", "forecast"},
|
kinds: []fkevent.Kind{fkevent.Kind(standards.KindObservation), fkevent.Kind(standards.KindForecast)},
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := fksources.ValidateExpectedKinds(sc, in); err != nil {
|
if err := fksources.ValidateExpectedKinds(sc, in); err != nil {
|
||||||
@@ -48,10 +49,10 @@ func TestValidateSourceExpectedKindsSubsetAllowed(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateSourceExpectedKindsMismatchFails(t *testing.T) {
|
func TestValidateSourceExpectedKindsMismatchFails(t *testing.T) {
|
||||||
sc := config.SourceConfig{Kinds: []string{"alert"}}
|
sc := config.SourceConfig{Kinds: []string{standards.KindAlert}}
|
||||||
in := testKindsSource{
|
in := testKindsSource{
|
||||||
testInput: testInput{name: "test"},
|
testInput: testInput{name: "test"},
|
||||||
kinds: []fkevent.Kind{"observation", "forecast"},
|
kinds: []fkevent.Kind{fkevent.Kind(standards.KindObservation), fkevent.Kind(standards.KindForecast)},
|
||||||
}
|
}
|
||||||
|
|
||||||
err := fksources.ValidateExpectedKinds(sc, in)
|
err := fksources.ValidateExpectedKinds(sc, in)
|
||||||
@@ -64,7 +65,7 @@ func TestValidateSourceExpectedKindsMismatchFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateSourceExpectedKindsNoMetadataSkipsCheck(t *testing.T) {
|
func TestValidateSourceExpectedKindsNoMetadataSkipsCheck(t *testing.T) {
|
||||||
sc := config.SourceConfig{Kinds: []string{"alert"}}
|
sc := config.SourceConfig{Kinds: []string{standards.KindAlert}}
|
||||||
in := testInput{name: "test"}
|
in := testInput{name: "test"}
|
||||||
|
|
||||||
if err := fksources.ValidateExpectedKinds(sc, in); err != nil {
|
if err := fksources.ValidateExpectedKinds(sc, in); err != nil {
|
||||||
@@ -111,6 +112,10 @@ func TestMaintainedConfigExamplesLoad(t *testing.T) {
|
|||||||
func assertConfigSourcesBuildSchedulerJobs(t *testing.T, cfg *config.Config) {
|
func assertConfigSourcesBuildSchedulerJobs(t *testing.T, cfg *config.Config) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
if len(cfg.Sources) == 0 {
|
||||||
|
t.Fatalf("config has no sources")
|
||||||
|
}
|
||||||
|
|
||||||
reg := fksources.NewRegistry()
|
reg := fksources.NewRegistry()
|
||||||
wfsources.RegisterBuiltins(reg)
|
wfsources.RegisterBuiltins(reg)
|
||||||
|
|
||||||
@@ -158,7 +163,7 @@ func TestNormalizeNoMatchPassThrough(t *testing.T) {
|
|||||||
pl := &fkpipeline.Pipeline{Processors: chain}
|
pl := &fkpipeline.Pipeline{Processors: chain}
|
||||||
in := fkevent.Event{
|
in := fkevent.Event{
|
||||||
ID: "evt-no-match",
|
ID: "evt-no-match",
|
||||||
Kind: fkevent.Kind("observation"),
|
Kind: fkevent.Kind(standards.KindObservation),
|
||||||
Source: "test",
|
Source: "test",
|
||||||
EmittedAt: time.Now().UTC(),
|
EmittedAt: time.Now().UTC(),
|
||||||
Schema: "raw.weatherfeeder.unknown.v1",
|
Schema: "raw.weatherfeeder.unknown.v1",
|
||||||
@@ -188,7 +193,7 @@ func TestDedupeDropsSecondEventWithSameID(t *testing.T) {
|
|||||||
pl := &fkpipeline.Pipeline{Processors: chain}
|
pl := &fkpipeline.Pipeline{Processors: chain}
|
||||||
in := fkevent.Event{
|
in := fkevent.Event{
|
||||||
ID: "evt-dedupe-1",
|
ID: "evt-dedupe-1",
|
||||||
Kind: fkevent.Kind("observation"),
|
Kind: fkevent.Kind(standards.KindObservation),
|
||||||
Source: "test",
|
Source: "test",
|
||||||
EmittedAt: time.Now().UTC(),
|
EmittedAt: time.Now().UTC(),
|
||||||
Schema: "raw.weatherfeeder.unknown.v1",
|
Schema: "raw.weatherfeeder.unknown.v1",
|
||||||
|
|||||||
@@ -147,7 +147,7 @@ URL omits it or sets another unit system.
|
|||||||
|
|
||||||
## SPC Convective Outlook Params
|
## SPC Convective Outlook Params
|
||||||
|
|
||||||
`spc_convective_outlook` fetches the twelve required Day 1-3 GeoJSON outlook
|
`spc_convective_outlook` fetches the nine required Day 1-3 GeoJSON outlook
|
||||||
products and the three required Day 1-3 print pages as one atomic bundle.
|
products and the three required Day 1-3 print pages as one atomic bundle.
|
||||||
|
|
||||||
| Param | Required | Description |
|
| Param | Required | Description |
|
||||||
@@ -165,8 +165,8 @@ products and the three required Day 1-3 print pages as one atomic bundle.
|
|||||||
|
|
||||||
GeoJSON product keys are `day1_categorical`, `day1_tornado`, `day1_hail`,
|
GeoJSON product keys are `day1_categorical`, `day1_tornado`, `day1_hail`,
|
||||||
`day1_wind`, `day2_categorical`, `day2_tornado`, `day2_hail`, `day2_wind`,
|
`day1_wind`, `day2_categorical`, `day2_tornado`, `day2_hail`, `day2_wind`,
|
||||||
`day3_categorical`, `day3_tornado`, `day3_hail`, and `day3_wind`.
|
and `day3_categorical`. SPC does not provide Day 3 tornado, hail, or wind
|
||||||
Discussion keys are `day1`, `day2`, and `day3`.
|
GeoJSON products. Discussion keys are `day1`, `day2`, and `day3`.
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
sources:
|
sources:
|
||||||
|
|||||||
92
docs/consumers/api.md
Normal file
92
docs/consumers/api.md
Normal file
@@ -0,0 +1,92 @@
|
|||||||
|
# Consumer API Guide
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This guide is for developers and LLM coding agents integrating `weatherfeeder`
|
||||||
|
from another Go codebase.
|
||||||
|
|
||||||
|
`weatherfeeder` is primarily a daemon, not an SDK. Its public integration
|
||||||
|
surface is intentionally narrow:
|
||||||
|
|
||||||
|
- `model`: canonical weather payload structs.
|
||||||
|
- `standards`: schema strings, event kind strings, and shared WMO constants.
|
||||||
|
- JSON event output from stdout and NATS sinks.
|
||||||
|
- Postgres tables written by the optional Postgres sink.
|
||||||
|
|
||||||
|
Packages under `internal/` are implementation details and are not public
|
||||||
|
integration surfaces.
|
||||||
|
|
||||||
|
## Recommended Workflow
|
||||||
|
|
||||||
|
Consumers should switch on the event `schema` value and decode `payload` into
|
||||||
|
the matching `model` type.
|
||||||
|
|
||||||
|
Minimal example:
|
||||||
|
|
||||||
|
```go
|
||||||
|
package consumer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Event struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Schema string `json:"schema"`
|
||||||
|
Payload json.RawMessage `json:"payload"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func Decode(payload []byte) (any, error) {
|
||||||
|
var evt Event
|
||||||
|
if err := json.Unmarshal(payload, &evt); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
switch evt.Schema {
|
||||||
|
case standards.SchemaWeatherObservationV1:
|
||||||
|
var out model.WeatherObservation
|
||||||
|
return &out, json.Unmarshal(evt.Payload, &out)
|
||||||
|
case standards.SchemaWeatherForecastV1:
|
||||||
|
var out model.WeatherForecastRun
|
||||||
|
return &out, json.Unmarshal(evt.Payload, &out)
|
||||||
|
case standards.SchemaWeatherForecastDiscussionV1:
|
||||||
|
var out model.WeatherForecastDiscussion
|
||||||
|
return &out, json.Unmarshal(evt.Payload, &out)
|
||||||
|
case standards.SchemaWeatherStoryV1:
|
||||||
|
var out model.WeatherStoryRun
|
||||||
|
return &out, json.Unmarshal(evt.Payload, &out)
|
||||||
|
case standards.SchemaWeatherAlertV1:
|
||||||
|
var out model.WeatherAlertRun
|
||||||
|
return &out, json.Unmarshal(evt.Payload, &out)
|
||||||
|
case standards.SchemaWeatherOutlookV2:
|
||||||
|
var out model.WeatherOutlookRun
|
||||||
|
return &out, json.Unmarshal(evt.Payload, &out)
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("unsupported weatherfeeder schema %q", evt.Schema)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Consumer Responsibilities
|
||||||
|
|
||||||
|
- Treat event IDs as opaque.
|
||||||
|
- Treat absent `omitempty` fields as unknown, not zero.
|
||||||
|
- Prefer schema constants from `standards` over string literals in Go code.
|
||||||
|
- Expect canonical numeric measurements to use metric units.
|
||||||
|
- Expect canonical timestamps from normalizers to be UTC unless a field-specific
|
||||||
|
contract says otherwise.
|
||||||
|
- Handle additive fields within the same schema version.
|
||||||
|
- Do not import `internal/...` packages.
|
||||||
|
|
||||||
|
## Canonical References
|
||||||
|
|
||||||
|
- Public payload package: [`pkg-model.md`](pkg-model.md).
|
||||||
|
- Public constants package: [`pkg-standards.md`](pkg-standards.md).
|
||||||
|
- JSON event wire contract: [`../integrations/events.md`](../integrations/events.md).
|
||||||
|
- Postgres table contract: [`../integrations/postgres.md`](../integrations/postgres.md).
|
||||||
|
- Runtime and adapter architecture: [`../policy/architecture.md`](../policy/architecture.md).
|
||||||
67
docs/consumers/pkg-model.md
Normal file
67
docs/consumers/pkg-model.md
Normal file
@@ -0,0 +1,67 @@
|
|||||||
|
# Package `model`
|
||||||
|
|
||||||
|
## Import Path
|
||||||
|
|
||||||
|
```go
|
||||||
|
import "gitea.maximumdirect.net/ejr/weatherfeeder/model"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Package `model` defines `weatherfeeder`'s canonical weather payload structs.
|
||||||
|
These structs are emitted as the `payload` of canonical `weather.*` events and
|
||||||
|
are also the domain types consumed by downstream applications such as
|
||||||
|
`weatherapi`.
|
||||||
|
|
||||||
|
The JSON field tags on these structs are part of the wire contract. For the full
|
||||||
|
field-by-field JSON contract, use the [event wire contract](../integrations/events.md).
|
||||||
|
|
||||||
|
## Payload Types
|
||||||
|
|
||||||
|
Current canonical schema families map to these public types:
|
||||||
|
|
||||||
|
| Schema | Primary type |
|
||||||
|
|---|---|
|
||||||
|
| `weather.observation.v1` | `WeatherObservation` |
|
||||||
|
| `weather.forecast.v1` | `WeatherForecastRun` |
|
||||||
|
| `weather.forecast_discussion.v1` | `WeatherForecastDiscussion` |
|
||||||
|
| `weather.weather_story.v1` | `WeatherStoryRun` |
|
||||||
|
| `weather.alert.v1` | `WeatherAlertRun` |
|
||||||
|
| `weather.outlook.v2` | `WeatherOutlookRun` |
|
||||||
|
|
||||||
|
Related child types include:
|
||||||
|
|
||||||
|
- `WeatherObservationPresentWeather`
|
||||||
|
- `WeatherForecastPeriod`
|
||||||
|
- `WeatherForecastDiscussionSection`
|
||||||
|
- `WeatherStory`
|
||||||
|
- `WeatherAlert`
|
||||||
|
- `WeatherAlertReference`
|
||||||
|
- `WeatherOutlook`
|
||||||
|
- `WeatherOutlookDiscussion`
|
||||||
|
- `WMOCode`
|
||||||
|
|
||||||
|
`WeatherOutlookRun` includes `WeatherOutlookDiscussion` entries as run-level
|
||||||
|
SPC day discussions.
|
||||||
|
|
||||||
|
## Wire And Compatibility Rules
|
||||||
|
|
||||||
|
- JSON tags define canonical payload field names.
|
||||||
|
- Pointer fields and fields tagged `omitempty` are optional on the wire.
|
||||||
|
- Missing optional fields mean unknown or not applicable.
|
||||||
|
- Canonical measurements use metric units.
|
||||||
|
- Canonical timestamps are `time.Time` values encoded by Go's JSON encoder.
|
||||||
|
- Normalized canonical timestamps are UTC unless a field-specific contract says
|
||||||
|
otherwise.
|
||||||
|
- Additive fields are compatible within a schema version.
|
||||||
|
- Removing, renaming, or changing the meaning of a field requires a new schema
|
||||||
|
identifier.
|
||||||
|
|
||||||
|
## Boundaries
|
||||||
|
|
||||||
|
`model` should not depend on source adapters, sinks, SQL column names, provider
|
||||||
|
HTTP shapes, or runtime configuration.
|
||||||
|
|
||||||
|
Consumers should not rely on packages under `internal/...`. Use `model` with
|
||||||
|
schema constants from [`standards`](pkg-standards.md) and the JSON contract in
|
||||||
|
[`docs/integrations/events.md`](../integrations/events.md).
|
||||||
96
docs/consumers/pkg-standards.md
Normal file
96
docs/consumers/pkg-standards.md
Normal file
@@ -0,0 +1,96 @@
|
|||||||
|
# Package `standards`
|
||||||
|
|
||||||
|
## Import Path
|
||||||
|
|
||||||
|
```go
|
||||||
|
import "gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Package `standards` defines stable identifiers and shared weather constants used
|
||||||
|
by `weatherfeeder` producers and Go consumers.
|
||||||
|
|
||||||
|
Use this package when switching on event schemas, comparing event kinds, or
|
||||||
|
working with canonical WMO condition codes.
|
||||||
|
|
||||||
|
## Event Kind Constants
|
||||||
|
|
||||||
|
Current event kind constants are:
|
||||||
|
|
||||||
|
| Constant | Value |
|
||||||
|
|---|---|
|
||||||
|
| `KindObservation` | `observation` |
|
||||||
|
| `KindForecast` | `forecast` |
|
||||||
|
| `KindForecastDiscussion` | `forecast_discussion` |
|
||||||
|
| `KindWeatherStory` | `weather_story` |
|
||||||
|
| `KindAlert` | `alert` |
|
||||||
|
| `KindOutlook` | `outlook` |
|
||||||
|
|
||||||
|
These are plain string constants. Convert them at adapter boundaries when using
|
||||||
|
feedkit's `event.Kind` type.
|
||||||
|
|
||||||
|
## Canonical Schema Constants
|
||||||
|
|
||||||
|
Canonical schemas emitted after normalization:
|
||||||
|
|
||||||
|
| Constant | Value |
|
||||||
|
|---|---|
|
||||||
|
| `SchemaWeatherObservationV1` | `weather.observation.v1` |
|
||||||
|
| `SchemaWeatherForecastV1` | `weather.forecast.v1` |
|
||||||
|
| `SchemaWeatherForecastDiscussionV1` | `weather.forecast_discussion.v1` |
|
||||||
|
| `SchemaWeatherStoryV1` | `weather.weather_story.v1` |
|
||||||
|
| `SchemaWeatherAlertV1` | `weather.alert.v1` |
|
||||||
|
| `SchemaWeatherOutlookV2` | `weather.outlook.v2` |
|
||||||
|
|
||||||
|
Historical canonical schema constant:
|
||||||
|
|
||||||
|
| Constant | Value |
|
||||||
|
|---|---|
|
||||||
|
| `SchemaWeatherOutlookV1` | `weather.outlook.v1` |
|
||||||
|
|
||||||
|
## Raw Schema Constants
|
||||||
|
|
||||||
|
Raw source schemas emitted by current registered sources:
|
||||||
|
|
||||||
|
| Constant | Value |
|
||||||
|
|---|---|
|
||||||
|
| `SchemaRawNWSObservationV1` | `raw.nws.observation.v1` |
|
||||||
|
| `SchemaRawOpenMeteoCurrentV1` | `raw.openmeteo.current.v1` |
|
||||||
|
| `SchemaRawOpenWeatherCurrentV1` | `raw.openweather.current.v1` |
|
||||||
|
| `SchemaRawNWSHourlyForecastV1` | `raw.nws.hourly.forecast.v1` |
|
||||||
|
| `SchemaRawNWSNarrativeForecastV1` | `raw.nws.narrative.forecast.v1` |
|
||||||
|
| `SchemaRawNWSForecastDiscussionV1` | `raw.nws.forecast_discussion.v1` |
|
||||||
|
| `SchemaRawNWSWeatherStoriesV1` | `raw.nws.weatherstories.v1` |
|
||||||
|
| `SchemaRawOpenMeteoHourlyForecastV1` | `raw.openmeteo.hourly.forecast.v1` |
|
||||||
|
| `SchemaRawNWSAlertsV1` | `raw.nws.alerts.v1` |
|
||||||
|
| `SchemaRawSPCConvectiveOutlookV1` | `raw.spc.convective_outlook.v1` |
|
||||||
|
|
||||||
|
Additional raw schema constant:
|
||||||
|
|
||||||
|
| Constant | Value |
|
||||||
|
|---|---|
|
||||||
|
| `SchemaRawOpenWeatherHourlyForecastV1` | `raw.openweather.hourly.forecast.v1` |
|
||||||
|
|
||||||
|
`SchemaRawOpenWeatherHourlyForecastV1` exists in code, but no current registered
|
||||||
|
source emits it. Consumers should not expect that raw schema unless a later
|
||||||
|
registered source documents it as part of the current event contract.
|
||||||
|
|
||||||
|
## WMO Constants And Text
|
||||||
|
|
||||||
|
`standards` also defines the canonical `WMOCode` vocabulary and text helpers
|
||||||
|
used by normalized observations and forecasts.
|
||||||
|
|
||||||
|
Consumer guidance:
|
||||||
|
|
||||||
|
- Treat `WMOUnknown` as unknown condition data.
|
||||||
|
- Observation `conditionCode` is required in the current event contract.
|
||||||
|
- Forecast period `conditionCode` is optional because some forecast products do
|
||||||
|
not provide a meaningful WMO condition.
|
||||||
|
- Prefer WMO constants and helper functions from this package instead of
|
||||||
|
duplicating code tables in consumers.
|
||||||
|
|
||||||
|
## Boundaries
|
||||||
|
|
||||||
|
`standards` is provider-agnostic. Provider-specific parsing belongs in
|
||||||
|
`weatherfeeder` internals, not in this package and not in consumers.
|
||||||
@@ -37,7 +37,7 @@ Canonical schemas emitted after normalization:
|
|||||||
| `forecast_discussion` | `weather.forecast_discussion.v1` | `WeatherForecastDiscussion` |
|
| `forecast_discussion` | `weather.forecast_discussion.v1` | `WeatherForecastDiscussion` |
|
||||||
| `weather_story` | `weather.weather_story.v1` | `WeatherStoryRun` |
|
| `weather_story` | `weather.weather_story.v1` | `WeatherStoryRun` |
|
||||||
| `alert` | `weather.alert.v1` | `WeatherAlertRun` |
|
| `alert` | `weather.alert.v1` | `WeatherAlertRun` |
|
||||||
| `outlook` | `weather.outlook.v1` | `WeatherOutlookRun` |
|
| `outlook` | `weather.outlook.v2` | `WeatherOutlookRun` |
|
||||||
|
|
||||||
Raw upstream schemas emitted by current sources:
|
Raw upstream schemas emitted by current sources:
|
||||||
|
|
||||||
@@ -220,13 +220,15 @@ Payload type: `WeatherAlertRun`.
|
|||||||
`references[]` entries contain optional `id`, `identifier`, `sender`, and
|
`references[]` entries contain optional `id`, `identifier`, `sender`, and
|
||||||
`sent` fields.
|
`sent` fields.
|
||||||
|
|
||||||
## `weather.outlook.v1`
|
## `weather.outlook.v2`
|
||||||
|
|
||||||
Payload type: `WeatherOutlookRun`.
|
Payload type: `WeatherOutlookRun`.
|
||||||
|
|
||||||
The current producer is the SPC convective outlook normalizer. It emits Day 1-3
|
The current producer is the SPC convective outlook normalizer. It emits Day 1-3
|
||||||
convective outlook polygons for categorical, tornado, hail, and wind products.
|
convective outlook snapshots for categorical, tornado, hail, and wind products
|
||||||
All timestamps are UTC.
|
that apply to the configured forecast point. Raw SPC bundles remain complete;
|
||||||
|
canonical outlook payloads are filtered to local polygons. All timestamps are
|
||||||
|
UTC.
|
||||||
|
|
||||||
| Field | Type | Required | Notes |
|
| Field | Type | Required | Notes |
|
||||||
|---|---|:---:|---|
|
|---|---|:---:|---|
|
||||||
@@ -236,7 +238,8 @@ All timestamps are UTC.
|
|||||||
| `longitude` | number | no | Configured point longitude in decimal degrees. |
|
| `longitude` | number | no | Configured point longitude in decimal degrees. |
|
||||||
| `asOf` | timestamp | yes | Snapshot time. For SPC, this is the latest outlook issue time when available. |
|
| `asOf` | timestamp | yes | Snapshot time. For SPC, this is the latest outlook issue time when available. |
|
||||||
| `issuedAt` | timestamp | no | Latest issue time across outlook features when any feature exists. |
|
| `issuedAt` | timestamp | no | Latest issue time across outlook features when any feature exists. |
|
||||||
| `outlooks` | array | yes | Ordered outlook polygons. |
|
| `outlooks` | array | yes | Ordered outlook polygons containing the configured point. |
|
||||||
|
| `discussions` | array | yes | Run-level day discussions for retained outlook days. |
|
||||||
|
|
||||||
`outlooks[]` entries:
|
`outlooks[]` entries:
|
||||||
|
|
||||||
@@ -255,19 +258,53 @@ All timestamps are UTC.
|
|||||||
| `issuedAt` | timestamp | yes | Feature issue time. |
|
| `issuedAt` | timestamp | yes | Feature issue time. |
|
||||||
| `expiresAt` | timestamp | yes | Expiration time; currently equal to `validTo`. |
|
| `expiresAt` | timestamp | yes | Expiration time; currently equal to `validTo`. |
|
||||||
| `forecaster` | string | no | SPC forecaster text, when present. |
|
| `forecaster` | string | no | SPC forecaster text, when present. |
|
||||||
| `headline` | string | no | Matching Day 1-3 print-page product title. |
|
|
||||||
| `summary` | string | no | Text from the print-page `...SUMMARY...` section. |
|
|
||||||
| `discussion` | string | no | Cleaned full print-page product text. |
|
|
||||||
| `sourceUrl` | string | no | GeoJSON product URL for this outlook feature. |
|
| `sourceUrl` | string | no | GeoJSON product URL for this outlook feature. |
|
||||||
| `imageUrl` | string | no | Reserved for provider image URLs; currently empty. |
|
| `imageUrl` | string | no | Reserved for provider image URLs; currently empty. |
|
||||||
| `containsLocation` | boolean | yes | Whether the configured point is inside or on the boundary of the polygon. |
|
| `containsLocation` | boolean | yes | Always `true` for emitted v2 outlooks. |
|
||||||
| `geometry` | object | yes | Compact GeoJSON `Polygon` or `MultiPolygon` geometry. |
|
| `geometry` | object | yes | Compact GeoJSON `Polygon` or `MultiPolygon` geometry. |
|
||||||
|
|
||||||
`geometry` preserves the SPC feature geometry as compact GeoJSON using
|
`geometry` preserves the SPC feature geometry as compact GeoJSON using
|
||||||
`[longitude, latitude]` coordinate order. `containsLocation` is computed with
|
`[longitude, latitude]` coordinate order. `containsLocation` is computed with
|
||||||
that geometry and the configured source `latitude`/`longitude`; boundary points
|
that geometry and the configured source `latitude`/`longitude`; boundary points
|
||||||
count as contained. All outlook polygons are emitted, including polygons that do
|
count as contained. Polygons that do not contain the configured point are not
|
||||||
not contain the configured point.
|
included in canonical v2 payloads.
|
||||||
|
|
||||||
|
`discussions[]` entries:
|
||||||
|
|
||||||
|
| Field | Type | Required | Notes |
|
||||||
|
|---|---|:---:|---|
|
||||||
|
| `day` | integer | yes | SPC outlook day, currently `1`, `2`, or `3`. |
|
||||||
|
| `headline` | string | no | Matching Day 1-3 print-page product title. |
|
||||||
|
| `summary` | string | no | Text from the print-page `...SUMMARY...` section. |
|
||||||
|
| `discussion` | string | no | Cleaned full print-page product text. |
|
||||||
|
| `updatedAt` | timestamp | no | Print-page update time, when present. |
|
||||||
|
|
||||||
|
When no SPC polygons apply locally, the run is still emitted with `outlooks: []`
|
||||||
|
and `discussions: []`. Discussions are included only for days represented by at
|
||||||
|
least one retained outlook, and multiple retained outlook types for the same day
|
||||||
|
share one discussion entry.
|
||||||
|
|
||||||
|
### SPC Outlook Supersession
|
||||||
|
|
||||||
|
Consumers should prefer latest-run semantics for current conditions: read the
|
||||||
|
latest `WeatherOutlookRun` for the configured location and use its `outlooks`
|
||||||
|
and `discussions` arrays together.
|
||||||
|
|
||||||
|
Historical SQL consumers that collapse older rows should identify superseded
|
||||||
|
outlooks by `provider`, `product`, `outlookType`, `validFrom`, and `validTo`,
|
||||||
|
then keep rows with the greatest `issuedAt`. `day` and `label` are not identity
|
||||||
|
fields. When multiple retained polygons share that latest `issuedAt`, preserve
|
||||||
|
the full group.
|
||||||
|
|
||||||
|
## Legacy `weather.outlook.v1`
|
||||||
|
|
||||||
|
`weather.outlook.v1` is a historical canonical schema retained as a standards
|
||||||
|
constant for older data and consumers. Current SPC normalization emits
|
||||||
|
`weather.outlook.v2`.
|
||||||
|
|
||||||
|
The v1 payload used `WeatherOutlookRun` and placed `headline`, `summary`, and
|
||||||
|
`discussion` on each `outlooks[]` polygon. It also represented the pre-v2 SPC
|
||||||
|
canonical behavior, where national polygons were preserved in canonical output.
|
||||||
|
|
||||||
## Compact Example
|
## Compact Example
|
||||||
|
|
||||||
|
|||||||
106
docs/integrations/feedkit.md
Normal file
106
docs/integrations/feedkit.md
Normal file
@@ -0,0 +1,106 @@
|
|||||||
|
# Feedkit Integration
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This document describes the feedkit runtime behavior that `weatherfeeder`
|
||||||
|
currently depends on. It is for maintainers and LLM coding agents changing
|
||||||
|
runtime wiring, config behavior, source construction, processing, routing, or
|
||||||
|
sink behavior.
|
||||||
|
|
||||||
|
Weather-domain behavior belongs in `weatherfeeder`. Generic daemon mechanics
|
||||||
|
belong to feedkit.
|
||||||
|
|
||||||
|
## Current Dependency
|
||||||
|
|
||||||
|
`weatherfeeder` imports feedkit as its daemon framework dependency. The exact
|
||||||
|
module version is declared in `go.mod`.
|
||||||
|
|
||||||
|
Feedkit provides:
|
||||||
|
|
||||||
|
- YAML config loading and validation.
|
||||||
|
- Source, processor, and sink registries.
|
||||||
|
- HTTP source helper behavior.
|
||||||
|
- Scheduler polling.
|
||||||
|
- Normalize and dedupe processors.
|
||||||
|
- Route compilation and sink dispatch.
|
||||||
|
- Built-in stdout, NATS, and Postgres sink mechanics.
|
||||||
|
|
||||||
|
## Config Contract
|
||||||
|
|
||||||
|
`cmd/weatherfeeder` calls feedkit config loading for `config.yml` in the current
|
||||||
|
working directory.
|
||||||
|
|
||||||
|
Implemented behavior relied on by weatherfeeder docs and tests:
|
||||||
|
|
||||||
|
- Top-level config contains `sources`, `sinks`, and optional `routes`.
|
||||||
|
- Config struct fields are decoded strictly, so misspelled struct fields fail
|
||||||
|
startup.
|
||||||
|
- Driver-specific `params` maps are decoded generically and validated by the
|
||||||
|
source or sink constructor that consumes them.
|
||||||
|
- Source `kinds` can be validated against a source's advertised `Kinds()`.
|
||||||
|
|
||||||
|
## Source And HTTP Contract
|
||||||
|
|
||||||
|
Most weatherfeeder sources use feedkit's single-document HTTP source helper for:
|
||||||
|
|
||||||
|
- request construction;
|
||||||
|
- `User-Agent` and `Accept` headers;
|
||||||
|
- optional conditional GET validators;
|
||||||
|
- response body size limits;
|
||||||
|
- context-aware HTTP work;
|
||||||
|
- unchanged `304 Not Modified` responses that emit no events.
|
||||||
|
|
||||||
|
The SPC convective outlook source fetches multiple documents itself, but it uses
|
||||||
|
feedkit transport helpers for HTTP clients and response body limits.
|
||||||
|
|
||||||
|
## Scheduler And Processing Contract
|
||||||
|
|
||||||
|
Weatherfeeder builds feedkit scheduler jobs from source configs. Current source
|
||||||
|
drivers are polling drivers and use the configured `every` interval.
|
||||||
|
|
||||||
|
Events flow through a feedkit pipeline in this order:
|
||||||
|
|
||||||
|
1. normalize processor;
|
||||||
|
2. dedupe processor.
|
||||||
|
|
||||||
|
The normalize processor is configured with `RequireMatch=false`, so unmatched
|
||||||
|
schemas pass through unchanged. Weatherfeeder registers its built-in normalizers
|
||||||
|
and owns the provider-to-canonical mapping.
|
||||||
|
|
||||||
|
The dedupe processor stores a bounded in-memory set of recent event IDs. The
|
||||||
|
bound is configured in `cmd/weatherfeeder`.
|
||||||
|
|
||||||
|
## Dispatch And Sink Contract
|
||||||
|
|
||||||
|
Feedkit compiles routes from config and dispatches processed events to matching
|
||||||
|
sinks. If `routes` is omitted, every configured sink receives every event kind.
|
||||||
|
|
||||||
|
Feedkit owns sink fanout mechanics, per-sink workers, queueing, context-aware
|
||||||
|
shutdown, and sink error logging. Weatherfeeder owns the event kinds and schemas
|
||||||
|
that make routes meaningful.
|
||||||
|
|
||||||
|
Built-in feedkit sinks used by weatherfeeder:
|
||||||
|
|
||||||
|
- `stdout`: validates and writes JSON events to stdout.
|
||||||
|
- `nats`: publishes JSON events to a configured subject.
|
||||||
|
- generic `postgres` sink factory: opens the database, ensures tables/indexes,
|
||||||
|
runs transactions, inserts mapped rows, and prunes when configured.
|
||||||
|
|
||||||
|
Weatherfeeder supplies its Postgres table schema and event mapper to feedkit's
|
||||||
|
Postgres sink factory. The table contract is documented in
|
||||||
|
[`postgres.md`](postgres.md).
|
||||||
|
|
||||||
|
## Boundaries
|
||||||
|
|
||||||
|
Do not move weather-domain policy into feedkit. Weatherfeeder owns:
|
||||||
|
|
||||||
|
- provider source drivers;
|
||||||
|
- raw and canonical schema constants;
|
||||||
|
- event kind meaning;
|
||||||
|
- canonical payload structs;
|
||||||
|
- normalizers;
|
||||||
|
- Postgres table shape and row mapping.
|
||||||
|
|
||||||
|
Do not duplicate generic feedkit mechanics in weatherfeeder unless there is a
|
||||||
|
narrow weather-specific reason. Runtime composition details are documented in
|
||||||
|
[`../internal/runtime.md`](../internal/runtime.md).
|
||||||
@@ -23,7 +23,7 @@ Events are mapped only for canonical weather schemas:
|
|||||||
- `weather.forecast_discussion.v1`
|
- `weather.forecast_discussion.v1`
|
||||||
- `weather.weather_story.v1`
|
- `weather.weather_story.v1`
|
||||||
- `weather.alert.v1`
|
- `weather.alert.v1`
|
||||||
- `weather.outlook.v1`
|
- `weather.outlook.v2`
|
||||||
|
|
||||||
Unsupported schemas produce no writes for this sink. Mapped events are inserted
|
Unsupported schemas produce no writes for this sink. Mapped events are inserted
|
||||||
transactionally. Inserts use ordinary `INSERT`; duplicate primary keys fail the
|
transactionally. Inserts use ordinary `INSERT`; duplicate primary keys fail the
|
||||||
@@ -59,6 +59,7 @@ Parent tables store the feed event envelope:
|
|||||||
| `alert_references` | `run_event_id`, `alert_index`, `reference_index` | `as_of` |
|
| `alert_references` | `run_event_id`, `alert_index`, `reference_index` | `as_of` |
|
||||||
| `outlook_runs` | `event_id` | `as_of` |
|
| `outlook_runs` | `event_id` | `as_of` |
|
||||||
| `outlooks` | `run_event_id`, `outlook_index` | `as_of` |
|
| `outlooks` | `run_event_id`, `outlook_index` | `as_of` |
|
||||||
|
| `outlook_discussions` | `run_event_id`, `discussion_index` | `as_of` |
|
||||||
|
|
||||||
## Table Contract
|
## Table Contract
|
||||||
|
|
||||||
@@ -408,6 +409,7 @@ Indexes:
|
|||||||
| `as_of` | `TIMESTAMPTZ` | no | `payload.asOf` |
|
| `as_of` | `TIMESTAMPTZ` | no | `payload.asOf` |
|
||||||
| `issued_at` | `TIMESTAMPTZ` | yes | `payload.issuedAt` |
|
| `issued_at` | `TIMESTAMPTZ` | yes | `payload.issuedAt` |
|
||||||
| `outlook_count` | `INTEGER` | no | `len(payload.outlooks)` |
|
| `outlook_count` | `INTEGER` | no | `len(payload.outlooks)` |
|
||||||
|
| `discussion_count` | `INTEGER` | no | `len(payload.discussions)` |
|
||||||
|
|
||||||
### `outlooks`
|
### `outlooks`
|
||||||
|
|
||||||
@@ -442,14 +444,36 @@ Indexes:
|
|||||||
| `issued_at` | `TIMESTAMPTZ` | no | `payload.outlooks[].issuedAt` |
|
| `issued_at` | `TIMESTAMPTZ` | no | `payload.outlooks[].issuedAt` |
|
||||||
| `expires_at` | `TIMESTAMPTZ` | no | `payload.outlooks[].expiresAt` |
|
| `expires_at` | `TIMESTAMPTZ` | no | `payload.outlooks[].expiresAt` |
|
||||||
| `forecaster` | `TEXT` | yes | `payload.outlooks[].forecaster` |
|
| `forecaster` | `TEXT` | yes | `payload.outlooks[].forecaster` |
|
||||||
| `headline` | `TEXT` | yes | `payload.outlooks[].headline` |
|
|
||||||
| `summary` | `TEXT` | yes | `payload.outlooks[].summary` |
|
|
||||||
| `discussion` | `TEXT` | yes | `payload.outlooks[].discussion` |
|
|
||||||
| `source_url` | `TEXT` | yes | `payload.outlooks[].sourceUrl` |
|
| `source_url` | `TEXT` | yes | `payload.outlooks[].sourceUrl` |
|
||||||
| `image_url` | `TEXT` | yes | `payload.outlooks[].imageUrl` |
|
| `image_url` | `TEXT` | yes | `payload.outlooks[].imageUrl` |
|
||||||
| `contains_location` | `BOOLEAN` | no | `payload.outlooks[].containsLocation` |
|
| `contains_location` | `BOOLEAN` | no | `payload.outlooks[].containsLocation` |
|
||||||
| `geometry_json` | `TEXT` | no | Compact JSON from `payload.outlooks[].geometry` |
|
| `geometry_json` | `TEXT` | no | Compact JSON from `payload.outlooks[].geometry` |
|
||||||
|
|
||||||
|
### `outlook_discussions`
|
||||||
|
|
||||||
|
Primary key: `run_event_id`, `discussion_index`
|
||||||
|
|
||||||
|
Prune column: `as_of`
|
||||||
|
|
||||||
|
Foreign key: `run_event_id` references `outlook_runs(event_id)` with cascade
|
||||||
|
delete.
|
||||||
|
|
||||||
|
Indexes:
|
||||||
|
|
||||||
|
- `idx_wf_outlook_discussions_day_as_of` on `day`, `as_of`
|
||||||
|
- unique `idx_wf_outlook_discussions_run_day` on `run_event_id`, `day`
|
||||||
|
|
||||||
|
| Column | Type | Null | Source |
|
||||||
|
|---|---|:---:|---|
|
||||||
|
| `run_event_id` | `TEXT REFERENCES outlook_runs(event_id) ON DELETE CASCADE` | no | Parent event ID. |
|
||||||
|
| `discussion_index` | `INTEGER` | no | `payload.discussions[]` index. |
|
||||||
|
| `as_of` | `TIMESTAMPTZ` | no | Parent `payload.asOf` |
|
||||||
|
| `day` | `INTEGER` | no | `payload.discussions[].day` |
|
||||||
|
| `headline` | `TEXT` | yes | `payload.discussions[].headline` |
|
||||||
|
| `summary` | `TEXT` | yes | `payload.discussions[].summary` |
|
||||||
|
| `discussion` | `TEXT` | yes | `payload.discussions[].discussion` |
|
||||||
|
| `updated_at` | `TIMESTAMPTZ` | yes | `payload.discussions[].updatedAt` |
|
||||||
|
|
||||||
## Retention
|
## Retention
|
||||||
|
|
||||||
When sink param `prune` is set, every successful write transaction deletes rows
|
When sink param `prune` is set, every successful write transaction deletes rows
|
||||||
@@ -472,5 +496,6 @@ binary does not provide CLI commands for them.
|
|||||||
- `WeatherAlertRun`: read `alert_runs`, join `alerts` by `run_event_id` ordered
|
- `WeatherAlertRun`: read `alert_runs`, join `alerts` by `run_event_id` ordered
|
||||||
by `alert_index`, then join `alert_references` by `run_event_id` and
|
by `alert_index`, then join `alert_references` by `run_event_id` and
|
||||||
`alert_index` ordered by `reference_index`.
|
`alert_index` ordered by `reference_index`.
|
||||||
- `WeatherOutlookRun`: read `outlook_runs`, then join `outlooks` by
|
- `WeatherOutlookRun`: read `outlook_runs`, join `outlooks` by `run_event_id`
|
||||||
`run_event_id` ordered by `outlook_index`.
|
ordered by `outlook_index`, then join `outlook_discussions` by
|
||||||
|
`run_event_id` ordered by `discussion_index`.
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ events are documented in [event wire contract](events.md).
|
|||||||
|
|
||||||
| Driver | Kind | Raw schema | Canonical schema |
|
| Driver | Kind | Raw schema | Canonical schema |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `spc_convective_outlook` | `outlook` | `raw.spc.convective_outlook.v1` | `weather.outlook.v1` |
|
| `spc_convective_outlook` | `outlook` | `raw.spc.convective_outlook.v1` | `weather.outlook.v2` |
|
||||||
|
|
||||||
## Config Requirements
|
## Config Requirements
|
||||||
|
|
||||||
@@ -39,11 +39,11 @@ current Day 1-3 SPC product URLs.
|
|||||||
|
|
||||||
## Upstream Products Used
|
## Upstream Products Used
|
||||||
|
|
||||||
The source fetches twelve required GeoJSON products every poll:
|
The source fetches nine required GeoJSON products every poll:
|
||||||
|
|
||||||
- Day 1 categorical, tornado, hail, and wind
|
- Day 1 categorical, tornado, hail, and wind
|
||||||
- Day 2 categorical, tornado, hail, and wind
|
- Day 2 categorical, tornado, hail, and wind
|
||||||
- Day 3 categorical, tornado, hail, and wind
|
- Day 3 categorical
|
||||||
|
|
||||||
It also fetches three required print pages:
|
It also fetches three required print pages:
|
||||||
|
|
||||||
@@ -52,8 +52,8 @@ It also fetches three required print pages:
|
|||||||
- Day 3 convective outlook print page
|
- Day 3 convective outlook print page
|
||||||
|
|
||||||
GeoJSON products are authoritative for outlook polygons, valid windows, issue
|
GeoJSON products are authoritative for outlook polygons, valid windows, issue
|
||||||
times, labels, and severity rank. Print pages are authoritative for headline,
|
times, labels, and severity rank. Print pages are authoritative for run-level
|
||||||
summary, and discussion text.
|
day discussion headline, summary, and discussion text.
|
||||||
|
|
||||||
## Accept Headers
|
## Accept Headers
|
||||||
|
|
||||||
@@ -99,23 +99,33 @@ Raw source `effective_at` prefers:
|
|||||||
4. fetch time.
|
4. fetch time.
|
||||||
|
|
||||||
The normalizer sets canonical `asOf` and normalized event `effective_at` from
|
The normalizer sets canonical `asOf` and normalized event `effective_at` from
|
||||||
the latest valid outlook feature `issuedAt`, with fallback to print-page update
|
the latest valid GeoJSON issue time across the complete raw bundle, including
|
||||||
time and then the incoming event envelope.
|
empty no-risk placeholders, with fallback to print-page update time and then the
|
||||||
|
incoming event envelope.
|
||||||
|
|
||||||
## Mapping Notes
|
## Mapping Notes
|
||||||
|
|
||||||
Each GeoJSON feature becomes one canonical outlook. Products are ordered by day,
|
The raw source fetches and envelopes the complete SPC bundle. The normalizer
|
||||||
then categorical, tornado, hail, and wind. Feature order is preserved within
|
decodes every configured GeoJSON product, skips empty no-risk
|
||||||
each product.
|
`GeometryCollection` placeholders, and emits canonical outlooks only when the
|
||||||
|
configured point is inside or on the boundary of a real feature. Products are
|
||||||
|
ordered by day, then categorical, tornado, hail, and wind. Retained feature order
|
||||||
|
is preserved within each product.
|
||||||
|
|
||||||
The normalizer computes `containsLocation` with the configured latitude and
|
The normalizer computes `containsLocation` with the configured latitude and
|
||||||
longitude against compact GeoJSON `Polygon` or `MultiPolygon` geometry.
|
longitude against compact GeoJSON `Polygon` or `MultiPolygon` geometry.
|
||||||
Coordinates use GeoJSON order, `[longitude, latitude]`, and boundary points
|
Coordinates use GeoJSON order, `[longitude, latitude]`, and boundary points
|
||||||
count as contained.
|
count as contained.
|
||||||
|
|
||||||
All outlook polygons are preserved, including polygons that do not contain the
|
Canonical outlook runs are emitted even when no polygons apply locally. In that
|
||||||
configured point. Matching print-page headline, summary, and discussion text is
|
case the payload contains empty `outlooks` and `discussions` arrays.
|
||||||
attached to every outlook for the same day.
|
|
||||||
|
Print-page prose is represented as run-level day discussions. Discussions are
|
||||||
|
included only for days represented by at least one retained outlook. Multiple
|
||||||
|
retained outlook types for the same day share one discussion entry.
|
||||||
|
|
||||||
|
For downstream current-state and historical supersession guidance, see the
|
||||||
|
[event wire contract](events.md#spc-outlook-supersession).
|
||||||
|
|
||||||
## Failure Behavior
|
## Failure Behavior
|
||||||
|
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ Current mappings:
|
|||||||
| `raw.nws.forecast_discussion.v1` | `weather.forecast_discussion.v1` |
|
| `raw.nws.forecast_discussion.v1` | `weather.forecast_discussion.v1` |
|
||||||
| `raw.nws.weatherstories.v1` | `weather.weather_story.v1` |
|
| `raw.nws.weatherstories.v1` | `weather.weather_story.v1` |
|
||||||
| `raw.nws.alerts.v1` | `weather.alert.v1` |
|
| `raw.nws.alerts.v1` | `weather.alert.v1` |
|
||||||
| `raw.spc.convective_outlook.v1` | `weather.outlook.v1` |
|
| `raw.spc.convective_outlook.v1` | `weather.outlook.v2` |
|
||||||
|
|
||||||
## Boundaries
|
## Boundaries
|
||||||
|
|
||||||
@@ -59,13 +59,13 @@ Weatherfeeder registers normalizers in a stable order:
|
|||||||
|
|
||||||
The current normalizers avoid ambiguous matches by using schema equality.
|
The current normalizers avoid ambiguous matches by using schema equality.
|
||||||
|
|
||||||
The SPC outlook normalizer decodes the raw multi-document bundle, maps each
|
The SPC outlook normalizer decodes the raw multi-document bundle, maps
|
||||||
GeoJSON feature to a canonical outlook, and enriches all outlooks for a day with
|
location-containing GeoJSON features to canonical outlooks, and adds one
|
||||||
the matching print-page headline, summary, and discussion. It preserves compact
|
run-level print-page discussion per retained outlook day. It preserves compact
|
||||||
GeoJSON feature geometry and computes `containsLocation` with
|
GeoJSON feature geometry and computes `containsLocation` with
|
||||||
`internal/geo.ContainsPoint` using the source-configured point. Boundary points
|
`internal/geo.ContainsPoint` using the source-configured point. Boundary points
|
||||||
count as contained, and all polygons are preserved whether or not they contain
|
count as contained. Polygons that do not contain the point are omitted from the
|
||||||
the point.
|
canonical run.
|
||||||
|
|
||||||
## State
|
## State
|
||||||
|
|
||||||
|
|||||||
@@ -19,11 +19,16 @@ Inputs are canonical feed events. The mapper currently handles these schemas:
|
|||||||
- `weather.forecast_discussion.v1`
|
- `weather.forecast_discussion.v1`
|
||||||
- `weather.weather_story.v1`
|
- `weather.weather_story.v1`
|
||||||
- `weather.alert.v1`
|
- `weather.alert.v1`
|
||||||
- `weather.outlook.v1`
|
- `weather.outlook.v2`
|
||||||
|
|
||||||
Outputs are feedkit `PostgresWrite` values for weatherfeeder-owned tables.
|
Outputs are feedkit `PostgresWrite` values for weatherfeeder-owned tables.
|
||||||
Unsupported schemas produce no writes and no error.
|
Unsupported schemas produce no writes and no error.
|
||||||
|
|
||||||
|
Outlook runs are written to `outlook_runs`, retained local polygons are written
|
||||||
|
to `outlooks`, and run-level day discussions are written to
|
||||||
|
`outlook_discussions`. The parent run row stores `outlook_count` and
|
||||||
|
`discussion_count`.
|
||||||
|
|
||||||
## Boundaries
|
## Boundaries
|
||||||
|
|
||||||
- Weatherfeeder owns table definitions in `schema.go`.
|
- Weatherfeeder owns table definitions in `schema.go`.
|
||||||
@@ -82,6 +87,7 @@ Child rows use positional indexes to preserve canonical array order:
|
|||||||
- `alert_index`
|
- `alert_index`
|
||||||
- `reference_index`
|
- `reference_index`
|
||||||
- `outlook_index`
|
- `outlook_index`
|
||||||
|
- `discussion_index`
|
||||||
|
|
||||||
Required canonical fields are validated before writes are returned:
|
Required canonical fields are validated before writes are returned:
|
||||||
|
|
||||||
@@ -94,7 +100,10 @@ Required canonical fields are validated before writes are returned:
|
|||||||
- alert runs require `asOf`, and each alert requires `id`;
|
- alert runs require `asOf`, and each alert requires `id`;
|
||||||
- outlook runs require `asOf`, and each outlook requires `id`, `provider`,
|
- outlook runs require `asOf`, and each outlook requires `id`, `provider`,
|
||||||
`product`, `day`, `outlookType`, `label`, `validFrom`, `validTo`, `issuedAt`,
|
`product`, `day`, `outlookType`, `label`, `validFrom`, `validTo`, `issuedAt`,
|
||||||
`expiresAt`, and `geometry`.
|
`expiresAt`, `containsLocation: true`, and `geometry`;
|
||||||
|
- outlook discussions require day `1`, `2`, or `3`; at least one of
|
||||||
|
`headline`, `summary`, or `discussion`; and no duplicate discussion day in the
|
||||||
|
same run.
|
||||||
|
|
||||||
Nullable canonical values are converted to SQL nulls by mapper helpers.
|
Nullable canonical values are converted to SQL nulls by mapper helpers.
|
||||||
Observation present-weather raw values and outlook geometry values are stored as
|
Observation present-weather raw values and outlook geometry values are stored as
|
||||||
|
|||||||
@@ -68,6 +68,21 @@ Runtime composition uses feedkit for:
|
|||||||
|
|
||||||
Weatherfeeder registers its own source drivers and its Postgres schema mapper.
|
Weatherfeeder registers its own source drivers and its Postgres schema mapper.
|
||||||
|
|
||||||
|
Responsibility split:
|
||||||
|
|
||||||
|
| Runtime concern | Owner |
|
||||||
|
| --- | --- |
|
||||||
|
| Config loading and generic validation | feedkit |
|
||||||
|
| Source, processor, and sink registries | feedkit mechanics; weatherfeeder registrations |
|
||||||
|
| Source polling and stream supervision | feedkit scheduler |
|
||||||
|
| Raw weather data fetching | weatherfeeder source adapters |
|
||||||
|
| Normalizer execution order and pass-through behavior | feedkit normalize processor |
|
||||||
|
| Weather raw-to-canonical mapping | weatherfeeder normalizers |
|
||||||
|
| Dedupe mechanics | feedkit dedupe processor |
|
||||||
|
| Route compilation and sink fanout | feedkit dispatch |
|
||||||
|
| Weather Postgres table shape and row mapping | weatherfeeder Postgres adapter |
|
||||||
|
| Postgres connection, DDL, inserts, transactions, and pruning | feedkit Postgres sink |
|
||||||
|
|
||||||
## State
|
## State
|
||||||
|
|
||||||
Weatherfeeder-owned runtime state is in process:
|
Weatherfeeder-owned runtime state is in process:
|
||||||
|
|||||||
@@ -82,7 +82,7 @@ document bodies as the previous successful poll.
|
|||||||
|
|
||||||
Every event passes through normalization first and dedupe second.
|
Every event passes through normalization first and dedupe second.
|
||||||
|
|
||||||
Normalizers match raw source schemas and produce canonical `weather.*.v1`
|
Normalizers match raw source schemas and produce versioned canonical `weather.*`
|
||||||
payloads. If an event has no matching normalizer, the normalize processor passes
|
payloads. If an event has no matching normalizer, the normalize processor passes
|
||||||
it through unchanged.
|
it through unchanged.
|
||||||
|
|
||||||
|
|||||||
@@ -23,10 +23,32 @@ The implemented runtime flow is:
|
|||||||
|
|
||||||
Canonical payload structs live in `model`. Schema identifiers and cross-provider wire conventions live in `standards`. Source adapters live under `internal/sources`. Normalizers live under `internal/normalizers`. Provider-specific parsing helpers shared by sources and normalizers live under `internal/providers`. Sink-specific persistence mapping lives under `internal/sinks`.
|
Canonical payload structs live in `model`. Schema identifiers and cross-provider wire conventions live in `standards`. Source adapters live under `internal/sources`. Normalizers live under `internal/normalizers`. Provider-specific parsing helpers shared by sources and normalizers live under `internal/providers`. Sink-specific persistence mapping lives under `internal/sinks`.
|
||||||
|
|
||||||
|
## Architecture Style
|
||||||
|
|
||||||
|
`weatherfeeder` uses a pragmatic ports-and-adapters architecture rather than a
|
||||||
|
formal framework. Provider APIs, config loading, scheduling, dispatch, and sinks
|
||||||
|
sit outside the weather domain model and normalization rules.
|
||||||
|
|
||||||
|
The implementation style is:
|
||||||
|
|
||||||
|
- Pipeline-oriented: events flow from source polling through normalization,
|
||||||
|
dedupe, routing, and sink fanout.
|
||||||
|
- Schema-routed: normalizers select raw payloads by explicit schema strings, not
|
||||||
|
source names or configured routes.
|
||||||
|
- Provider-isolated: NWS, Open-Meteo, OpenWeather, and SPC quirks stay in
|
||||||
|
provider-specific source, provider-helper, and normalizer packages.
|
||||||
|
- Registry-based: built-in source drivers, normalizers, processors, and sinks
|
||||||
|
are assembled explicitly through registries instead of dynamic plugin loading.
|
||||||
|
- Adapter-clean: persistence and external-system details stay behind source and
|
||||||
|
sink adapters, not in `model` or normalizers.
|
||||||
|
- Direct Go: prefer small package-level constructors and straightforward code
|
||||||
|
over broad abstractions.
|
||||||
|
|
||||||
## Core Design Principles
|
## Core Design Principles
|
||||||
|
|
||||||
- Hexagonal boundaries: provider APIs, config loading, scheduling, dispatch, and sinks are external mechanisms around the weather domain model and normalization logic.
|
- Hexagonal boundaries: provider APIs, config loading, scheduling, dispatch, and sinks are external mechanisms around the weather domain model and normalization logic.
|
||||||
- Raw-to-canonical flow: sources should fetch and envelope raw provider payloads; normalizers should own provider-to-canonical mapping.
|
- Raw-to-canonical flow: sources should fetch and envelope raw provider payloads; normalizers should own provider-to-canonical mapping.
|
||||||
|
- Location-focused canonical data: canonical weather events represent data relevant to the configured forecast location or configured provider object; `weatherfeeder` is not a national provider-data archive.
|
||||||
- Schema-based routing: normalizers match on event schema, not source name or event kind.
|
- Schema-based routing: normalizers match on event schema, not source name or event kind.
|
||||||
- Composable registries: source drivers, normalizers, processors, and sinks are assembled explicitly through registries.
|
- Composable registries: source drivers, normalizers, processors, and sinks are assembled explicitly through registries.
|
||||||
- Bounded concurrency: scheduling and sink fanout are concurrent, but the application should keep queues, goroutine ownership, logging, and cancellation behavior visible.
|
- Bounded concurrency: scheduling and sink fanout are concurrent, but the application should keep queues, goroutine ownership, logging, and cancellation behavior visible.
|
||||||
@@ -57,6 +79,23 @@ Tests and examples:
|
|||||||
- The sample `cmd/weatherfeeder/config.yml` is executable test input and is load-tested.
|
- The sample `cmd/weatherfeeder/config.yml` is executable test input and is load-tested.
|
||||||
- Tests should keep exercising package contracts directly rather than relying only on full-daemon execution.
|
- Tests should keep exercising package contracts directly rather than relying only on full-daemon execution.
|
||||||
|
|
||||||
|
## Feedkit Boundary
|
||||||
|
|
||||||
|
`feedkit` provides reusable daemon infrastructure. `weatherfeeder` provides the
|
||||||
|
weather-domain adapters, models, schemas, and mapping policy.
|
||||||
|
|
||||||
|
| Area | Feedkit owns | Weatherfeeder owns |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Config | Generic YAML shape: sources, sinks, routes, modes, cadence, and params. | Driver-specific config rules such as NWS `user_agent`, OpenWeather `units=metric`, and SPC coordinates. |
|
||||||
|
| Events | Domain-agnostic event envelope: ID, kind, source, emitted/effective times, schema, and payload. | Event kind meaning, schema strings, and canonical weather payloads. |
|
||||||
|
| Sources | Source interfaces, registry, expected-kind validation, HTTP helper, and default event ID helper. | NWS/Open-Meteo/OpenWeather/SPC source drivers and raw schema emission. |
|
||||||
|
| Processing | Processor registry, normalize processor, dedupe processor, and pipeline execution. | Weather normalizers and schema-specific raw-to-canonical mapping. |
|
||||||
|
| Dispatch | Route compilation and sink fanout mechanics. | Which weather event kinds are configured and meaningful. |
|
||||||
|
| Sinks | Generic stdout, NATS, and Postgres sink mechanics. | Weather-specific Postgres schema and canonical event-to-row mapping. |
|
||||||
|
|
||||||
|
Do not move weather-domain policy into `feedkit`, and do not duplicate generic
|
||||||
|
daemon mechanics in `weatherfeeder` when feedkit already provides the boundary.
|
||||||
|
|
||||||
## Modules Or Processing Steps
|
## Modules Or Processing Steps
|
||||||
|
|
||||||
The implemented processing steps are source polling, normalization, dedupe, and sink dispatch.
|
The implemented processing steps are source polling, normalization, dedupe, and sink dispatch.
|
||||||
|
|||||||
@@ -24,7 +24,8 @@ docs, not here.
|
|||||||
normalizers.
|
normalizers.
|
||||||
- `internal/sinks/postgres/`: weatherfeeder-owned Postgres schema and canonical
|
- `internal/sinks/postgres/`: weatherfeeder-owned Postgres schema and canonical
|
||||||
event mapper.
|
event mapper.
|
||||||
- `docs/`: current behavior, policies, integration contracts, and roadmap files.
|
- `docs/`: current behavior, consumer guides, integration contracts, policies,
|
||||||
|
and roadmap files.
|
||||||
- `examples/`: maintained, copyable configuration examples.
|
- `examples/`: maintained, copyable configuration examples.
|
||||||
|
|
||||||
## Build And Test
|
## Build And Test
|
||||||
@@ -70,6 +71,32 @@ Use fixtures, local test servers, and package-level tests.
|
|||||||
- Prefer explicit registries and small package-level constructors over hidden
|
- Prefer explicit registries and small package-level constructors over hidden
|
||||||
global behavior.
|
global behavior.
|
||||||
|
|
||||||
|
## Architecture-Preserving Changes
|
||||||
|
|
||||||
|
When changing `weatherfeeder`, preserve the split between feedkit
|
||||||
|
infrastructure and weather-domain behavior.
|
||||||
|
|
||||||
|
Do:
|
||||||
|
|
||||||
|
- keep generic scheduling, dispatch, processor, config, and sink mechanics in
|
||||||
|
feedkit;
|
||||||
|
- keep weather provider rules in source adapters, provider helpers, and
|
||||||
|
normalizers;
|
||||||
|
- keep canonical weather payloads in `model` and schema/wire identifiers in
|
||||||
|
`standards`;
|
||||||
|
- keep Postgres table and row mapping under `internal/sinks/postgres`;
|
||||||
|
- use explicit registries for built-in sources and normalizers.
|
||||||
|
|
||||||
|
Do not:
|
||||||
|
|
||||||
|
- move provider parsing, WMO mapping, or canonical weather policy into
|
||||||
|
`cmd/weatherfeeder`;
|
||||||
|
- move weather-specific constants, schemas, or validation rules into feedkit;
|
||||||
|
- put database column metadata or sink-specific tags on canonical model structs;
|
||||||
|
- replace explicit registries with dynamic plugin loading;
|
||||||
|
- introduce broad abstractions when a small provider-specific helper preserves
|
||||||
|
clarity.
|
||||||
|
|
||||||
## Dependency Policy
|
## Dependency Policy
|
||||||
|
|
||||||
Prefer the Go standard library unless a dependency materially improves
|
Prefer the Go standard library unless a dependency materially improves
|
||||||
@@ -183,6 +210,7 @@ When behavior changes, update the canonical docs in the same change:
|
|||||||
- CLI behavior: `docs/cli.md`;
|
- CLI behavior: `docs/cli.md`;
|
||||||
- operations and recovery: `docs/operations.md`;
|
- operations and recovery: `docs/operations.md`;
|
||||||
- troubleshooting: `docs/troubleshooting.md`;
|
- troubleshooting: `docs/troubleshooting.md`;
|
||||||
|
- public Go package consumption: `docs/consumers/`;
|
||||||
- external contracts: `docs/integrations/`;
|
- external contracts: `docs/integrations/`;
|
||||||
- internal component behavior: `docs/internal/`;
|
- internal component behavior: `docs/internal/`;
|
||||||
- copyable configs: `examples/`.
|
- copyable configs: `examples/`.
|
||||||
|
|||||||
@@ -2,12 +2,13 @@
|
|||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
Project documentation must help four audiences:
|
Project documentation must help five audiences:
|
||||||
|
|
||||||
1. users who need to run the application;
|
1. users who need to run the application;
|
||||||
2. administrators/operators who need to configure and operate it;
|
2. administrators/operators who need to configure and operate it;
|
||||||
3. developers who need to understand and change it safely;
|
3. developers who need to understand and change it safely;
|
||||||
4. LLM coding agents that need clear scope, boundaries, and invariants.
|
4. LLM coding agents that need clear scope, boundaries, and invariants;
|
||||||
|
5. developers and LLM coding agents integrating this project from another codebase.
|
||||||
|
|
||||||
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
|
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
|
||||||
|
|
||||||
@@ -42,11 +43,14 @@ Canonical homes:
|
|||||||
|
|
||||||
- project purpose and quickstart: `README.md`
|
- project purpose and quickstart: `README.md`
|
||||||
- development principles: `docs/policy/architecture.md`
|
- development principles: `docs/policy/architecture.md`
|
||||||
|
- public HTTP API reference: `docs/api.md`
|
||||||
- configuration reference: `docs/config.md`
|
- configuration reference: `docs/config.md`
|
||||||
- CLI reference: `docs/cli.md`
|
- CLI reference: `docs/cli.md`
|
||||||
- operations and recovery: `docs/operations.md`
|
- operations and recovery: `docs/operations.md`
|
||||||
- troubleshooting: `docs/troubleshooting.md`
|
- troubleshooting: `docs/troubleshooting.md`
|
||||||
|
- public API/package consumer guidance: `docs/consumers/`
|
||||||
- implemented internals: `docs/internal/`
|
- implemented internals: `docs/internal/`
|
||||||
|
- external protocol, service, and file-format contracts: `docs/integrations/`
|
||||||
- future work: `docs/roadmap/`
|
- future work: `docs/roadmap/`
|
||||||
- contributor workflow: `docs/policy/development.md`
|
- contributor workflow: `docs/policy/development.md`
|
||||||
- copyable examples: `examples/`
|
- copyable examples: `examples/`
|
||||||
@@ -106,7 +110,7 @@ Recommended:
|
|||||||
- `examples/`
|
- `examples/`
|
||||||
- `docs/policy/development.md`
|
- `docs/policy/development.md`
|
||||||
|
|
||||||
### Modular, staged, service-oriented, or orchestration application
|
### Modular, service-oriented, or orchestration application
|
||||||
|
|
||||||
Required:
|
Required:
|
||||||
- `docs/cli.md`, if CLI-based
|
- `docs/cli.md`, if CLI-based
|
||||||
@@ -119,6 +123,31 @@ Recommended:
|
|||||||
- `docs/troubleshooting.md`
|
- `docs/troubleshooting.md`
|
||||||
- validated examples under `examples/`
|
- validated examples under `examples/`
|
||||||
|
|
||||||
|
### Public HTTP API service
|
||||||
|
|
||||||
|
Required:
|
||||||
|
- `docs/api.md`
|
||||||
|
- `docs/cli.md`, if CLI-based
|
||||||
|
- `docs/config.md`, if config-driven
|
||||||
|
- `docs/operations.md`
|
||||||
|
- `docs/internal/`
|
||||||
|
- `docs/policy/development.md`
|
||||||
|
|
||||||
|
Recommended:
|
||||||
|
- `docs/troubleshooting.md`
|
||||||
|
- `docs/consumers/`, for task-oriented client integration guides
|
||||||
|
- `docs/integrations/`, for upstream/downstream service contracts
|
||||||
|
- validated examples under `examples/`
|
||||||
|
|
||||||
|
### Project with public packages or consumer APIs
|
||||||
|
|
||||||
|
Required:
|
||||||
|
- `docs/consumers/api.md`
|
||||||
|
- one `docs/consumers/pkg-<name>.md` file per public package, if public packages exist
|
||||||
|
|
||||||
|
Recommended:
|
||||||
|
- copyable consumer examples under `examples/`, if practical
|
||||||
|
|
||||||
## Required Documents
|
## Required Documents
|
||||||
|
|
||||||
### README.md
|
### README.md
|
||||||
@@ -161,6 +190,32 @@ It should include:
|
|||||||
|
|
||||||
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light.
|
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light.
|
||||||
|
|
||||||
|
### docs/api.md
|
||||||
|
|
||||||
|
**Audience:** external HTTP API consumers, developers, LLM coding agents integrating by HTTP
|
||||||
|
|
||||||
|
Required for projects whose primary public interface is HTTP.
|
||||||
|
|
||||||
|
`docs/api.md` is the canonical public HTTP API contract. It should be normative for external consumers and should not be duplicated by README, operations docs, consumer guides, or integration docs.
|
||||||
|
|
||||||
|
It should include:
|
||||||
|
|
||||||
|
1. base URL conventions;
|
||||||
|
2. authentication and authorization behavior, if implemented;
|
||||||
|
3. response envelope;
|
||||||
|
4. supported media types and content negotiation behavior;
|
||||||
|
5. shared query parameters;
|
||||||
|
6. endpoint reference grouped by route family;
|
||||||
|
7. request parameters and validation rules;
|
||||||
|
8. response fields, units, nullability, and optionality;
|
||||||
|
9. error response shape and status codes;
|
||||||
|
10. pagination, caching, rate-limit, idempotency, and retry behavior, if implemented;
|
||||||
|
11. compact request and response examples.
|
||||||
|
|
||||||
|
It must document only implemented endpoints and behavior. Planned endpoints, proposed fields, future filters, and experimental response shapes belong only under `docs/roadmap/`.
|
||||||
|
|
||||||
|
For HTTP API projects, `docs/consumers/` may provide task-oriented client integration guides, but those guides should link to `docs/api.md` for the authoritative endpoint contract.
|
||||||
|
|
||||||
### docs/policy/development.md
|
### docs/policy/development.md
|
||||||
|
|
||||||
**Audience:** developers, LLM coding agents
|
**Audience:** developers, LLM coding agents
|
||||||
@@ -175,7 +230,7 @@ It should include:
|
|||||||
- dependency policy;
|
- dependency policy;
|
||||||
- how to add config fields;
|
- how to add config fields;
|
||||||
- how to add CLI flags;
|
- how to add CLI flags;
|
||||||
- how to add stages/modules/adapters, if applicable;
|
- how to add modules or adapters, if applicable;
|
||||||
- how to update examples;
|
- how to update examples;
|
||||||
- documentation update expectations.
|
- documentation update expectations.
|
||||||
|
|
||||||
@@ -216,7 +271,7 @@ Explain when commands are useful, not just their syntax.
|
|||||||
|
|
||||||
**Audience:** administrators, operators
|
**Audience:** administrators, operators
|
||||||
|
|
||||||
Required for applications that maintain state, support resume behavior, run multiple stages, write durable artifacts, use remote storage, or require recovery procedures.
|
Required for applications that maintain state, support resume behavior, run multi-step workflows, write durable artifacts, use remote storage, or require recovery procedures.
|
||||||
|
|
||||||
It should cover:
|
It should cover:
|
||||||
|
|
||||||
@@ -244,11 +299,40 @@ Each entry should include:
|
|||||||
- safe fix;
|
- safe fix;
|
||||||
- relevant links.
|
- relevant links.
|
||||||
|
|
||||||
|
### docs/consumers/
|
||||||
|
|
||||||
|
**Audience:** developers and LLM coding agents integrating this project from another codebase
|
||||||
|
|
||||||
|
Required for projects with public packages, SDKs, client APIs, plugin APIs, or other application-facing integration surfaces.
|
||||||
|
|
||||||
|
This directory describes how an external codebase should consume the project's public API. It should be task-oriented and copyable where useful. It is not the place for internal implementation details or operator procedures.
|
||||||
|
|
||||||
|
For projects whose public API is HTTP, `docs/consumers/` is not required, and it should not duplicate the endpoint reference in `docs/api.md`. If present, it may provide practical integration workflows, client-specific examples, or migration notes that link back to `docs/api.md`.
|
||||||
|
|
||||||
|
`docs/consumers/api.md` should provide the consumer-facing overview and primary implementation workflow. It should include:
|
||||||
|
|
||||||
|
1. intended consumer audience and use cases;
|
||||||
|
2. required inputs supplied by operators or deployment configuration;
|
||||||
|
3. recommended public package or API workflow;
|
||||||
|
4. minimal copyable example;
|
||||||
|
5. consumer responsibilities and boundaries;
|
||||||
|
6. retry, idempotency, or status behavior, if applicable;
|
||||||
|
7. links to package-specific docs and canonical integration contracts.
|
||||||
|
|
||||||
|
Package-specific docs should be named `pkg-<name>.md` and should include:
|
||||||
|
|
||||||
|
1. import path;
|
||||||
|
2. intended use cases;
|
||||||
|
3. primary types and functions needed by consumers;
|
||||||
|
4. minimal examples;
|
||||||
|
5. validation, error, retry, and boundary behavior;
|
||||||
|
6. links to canonical file-format or wire-protocol contracts.
|
||||||
|
|
||||||
### docs/internal/
|
### docs/internal/
|
||||||
|
|
||||||
**Audience:** developers, LLM coding agents
|
**Audience:** developers, LLM coding agents
|
||||||
|
|
||||||
Required for modular, staged, service-oriented, or orchestration projects.
|
Required for modular, service-oriented, or orchestration projects.
|
||||||
|
|
||||||
This directory describes implemented internal components. It is not the roadmap.
|
This directory describes implemented internal components. It is not the roadmap.
|
||||||
|
|
||||||
@@ -289,7 +373,9 @@ Roadmap docs should not be confused with current behavior.
|
|||||||
|
|
||||||
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
|
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
|
||||||
|
|
||||||
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses.
|
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses or exposes.
|
||||||
|
|
||||||
|
For public HTTP API services, `docs/integrations/` should document upstream, downstream, storage, protocol, or runtime contracts that the service depends on or bridges. It should not become a second copy of the public HTTP endpoint reference; that belongs in `docs/api.md`.
|
||||||
|
|
||||||
Use one file per integration where useful.
|
Use one file per integration where useful.
|
||||||
|
|
||||||
@@ -346,8 +432,10 @@ Before merging documentation changes, verify:
|
|||||||
|
|
||||||
- README is concise and orientation-focused.
|
- README is concise and orientation-focused.
|
||||||
- `docs/policy/architecture.md` describes development principles.
|
- `docs/policy/architecture.md` describes development principles.
|
||||||
|
- `docs/api.md` is the canonical HTTP contract for HTTP API services.
|
||||||
- Future work appears only under `docs/roadmap/`.
|
- Future work appears only under `docs/roadmap/`.
|
||||||
- User-facing docs avoid unnecessary internals.
|
- User-facing docs avoid unnecessary internals.
|
||||||
|
- Consumer-facing docs explain public APIs without duplicating HTTP endpoint or integration contracts.
|
||||||
- Developer-facing docs preserve boundaries and invariants.
|
- Developer-facing docs preserve boundaries and invariants.
|
||||||
- Config examples match the schema.
|
- Config examples match the schema.
|
||||||
- CLI examples match real commands and flags.
|
- CLI examples match real commands and flags.
|
||||||
|
|||||||
@@ -1,558 +0,0 @@
|
|||||||
# Code Quality And Deduplication Audit
|
|
||||||
|
|
||||||
## Executive Summary
|
|
||||||
|
|
||||||
`weatherfeeder` is in good shape for a limited cleanup pass before the next major release. The current architecture is coherent: runtime composition is thin, provider fetching lives in source adapters, provider-specific parsing lives under `internal/providers`, canonical mapping lives in normalizers, and Postgres persistence is isolated under `internal/sinks/postgres`.
|
|
||||||
|
|
||||||
The codebase does not show a major architectural risk that would require broad redesign. Most duplication is the predictable result of recent feature growth across sources, canonical payloads, and persistence tables. The highest-value cleanup should be narrow and behavior-preserving.
|
|
||||||
|
|
||||||
Top refactoring targets:
|
|
||||||
|
|
||||||
1. Source adapter HTTP/config scaffolding: single-document sources and the SPC multi-document source share config, request, effective-time, and event-envelope policy in similar but not identical forms.
|
|
||||||
2. Postgres mapper boilerplate: parent envelope columns, UTC/null conversion, required-field checks, and child-row construction are repeated across every canonical product mapper.
|
|
||||||
3. Event kind and driver-name strings: event kinds and source driver names are repeated across registries, source implementations, tests, examples, and docs without code constants comparable to the centralized schema constants.
|
|
||||||
|
|
||||||
Recommended posture: perform a limited cleanup pass in small commits. Avoid broad framework changes, generic workflow engines, plugin systems, or ORM-like abstractions.
|
|
||||||
|
|
||||||
## Repository Map Reviewed
|
|
||||||
|
|
||||||
Inspected directories and packages:
|
|
||||||
|
|
||||||
- `cmd/weatherfeeder`: runtime composition, sample config, config-load and pipeline tests.
|
|
||||||
- `internal/sources`: source registry and provider source adapters for NWS, Open-Meteo, OpenWeather, and SPC.
|
|
||||||
- `internal/providers`: provider-specific parsing helpers for NWS, Open-Meteo, OpenWeather, and SPC.
|
|
||||||
- `internal/normalizers`: built-in normalizer registration, common helpers, and provider normalizers.
|
|
||||||
- `internal/sinks/postgres`: weatherfeeder-owned table schema and canonical-event mapper.
|
|
||||||
- `internal/geo`: point-in-geometry helper used by SPC outlook normalization.
|
|
||||||
- `model`: canonical payload structs.
|
|
||||||
- `standards`: schema and WMO constants.
|
|
||||||
- `examples`: maintained config examples.
|
|
||||||
- `docs`: policy, config, CLI, operations, troubleshooting, internal docs, integration docs, and `docs/roadmap/future.md`.
|
|
||||||
|
|
||||||
Major execution paths reviewed:
|
|
||||||
|
|
||||||
- daemon startup from `cmd/weatherfeeder/main.go`;
|
|
||||||
- source driver registration and source construction;
|
|
||||||
- source polling for single-document HTTP products and SPC multi-document bundles;
|
|
||||||
- normalizer registration and raw-schema dispatch;
|
|
||||||
- raw-to-canonical mapping for observations, forecasts, discussions, weather stories, alerts, and outlooks;
|
|
||||||
- Postgres schema and write mapping;
|
|
||||||
- maintained config loading tests.
|
|
||||||
|
|
||||||
Important areas not deeply inspected:
|
|
||||||
|
|
||||||
- Feedkit internals were not audited because they are a dependency and outside this repository's ownership boundary.
|
|
||||||
- Live upstream service behavior was not tested; this audit used local source inspection, fixtures, and existing docs.
|
|
||||||
- Full test execution was not run because this is a report-only task and no code behavior changed.
|
|
||||||
|
|
||||||
## High-Confidence Deduplication Opportunities
|
|
||||||
|
|
||||||
### 1. Centralize Repeated Source HTTP And Config Scaffolding
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/sources/nws/observation.go`
|
|
||||||
- `internal/sources/nws/alerts.go`
|
|
||||||
- `internal/sources/nws/forecast_common.go`
|
|
||||||
- `internal/sources/nws/forecast_discussion.go`
|
|
||||||
- `internal/sources/nws/weatherstories.go`
|
|
||||||
- `internal/sources/openmeteo/observation.go`
|
|
||||||
- `internal/sources/openmeteo/forecast.go`
|
|
||||||
- `internal/sources/openweather/observation.go`
|
|
||||||
- `internal/sources/spc/convective_outlook.go`
|
|
||||||
- `docs/internal/sources.md`
|
|
||||||
- `docs/config.md`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- Most sources wrap `fksources.NewHTTPSource`, implement `Name`, advertise one `Kinds` value, fetch raw content if changed, compute `effectiveAt`, call `DefaultEventID`, and emit a single event.
|
|
||||||
- The SPC source cannot use `HTTPSource` directly because it fetches an atomic multi-document bundle, but it repeats the same user-agent, HTTP timeout, body limit, request accept header, and unchanged-content policy at a lower level through `transport.FetchBodyWithLimit`.
|
|
||||||
- HTTP config params are documented as shared, but source construction has no weatherfeeder-owned helper for the common param names and error shape when a source cannot use `HTTPSource`.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- Adding future providers or multi-document products increases the chance of drift in `user_agent`, `http_timeout`, `http_response_body_limit_bytes`, body-limit, and error-message behavior.
|
|
||||||
- The single-document and SPC paths both implement operator-facing HTTP policy, but the shared policy is visible only in docs and feedkit conventions.
|
|
||||||
- Bug fixes to source envelope construction or shared HTTP params would likely need multiple package edits.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Add a small source-internal helper package or file, for example `internal/sources/sourceconfig` or `internal/sources/internal/httpconfig`, that owns weatherfeeder-specific HTTP param extraction for sources that cannot directly use `fksources.NewHTTPSource`.
|
|
||||||
- Keep `fksources.NewHTTPSource` as the implementation for simple sources. Do not replace it with a custom framework.
|
|
||||||
- Add a helper for common single-event envelope construction only if it remains explicit about `kind`, `source`, `schema`, `eventID`, `emittedAt`, `effectiveAt`, and payload. Avoid hiding provider-specific effective-time selection.
|
|
||||||
- For SPC, replace local parsing of `user_agent`, `http_timeout`, and `http_response_body_limit_bytes` with the shared helper while preserving its atomic multi-fetch and hash semantics.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- Add helper-level tests for param aliases, positive timeout/body-limit validation, missing `user_agent`, and error messages.
|
|
||||||
- Keep existing source tests for emitted kind/schema/effective time unchanged.
|
|
||||||
- Add one SPC constructor test that proves shared timeout/body-limit validation still applies.
|
|
||||||
|
|
||||||
Risk level: Low to Medium. The behavior is straightforward, but source constructor errors are user-facing and should be protected by tests.
|
|
||||||
|
|
||||||
### 2. Reduce Postgres Mapper Boilerplate Without Creating An ORM
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/sinks/postgres/map.go`
|
|
||||||
- `internal/sinks/postgres/schema.go`
|
|
||||||
- `internal/sinks/postgres/map_test.go`
|
|
||||||
- `internal/sinks/postgres/schema_test.go`
|
|
||||||
- `docs/internal/postgres-sink.md`
|
|
||||||
- `docs/integrations/postgres.md`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- Every parent table mapper repeats the same event envelope columns: `event_id`, `event_kind`, `event_source`, `event_schema`, `event_emitted_at`, and `event_effective_at`.
|
|
||||||
- Every run mapper follows the same pattern: decode payload, validate required run time/product fields, normalize to UTC, write one parent row, then write child rows with positional indexes.
|
|
||||||
- Nullable conversion helpers already exist, but each mapper repeats the same map literal shape and required-field error phrasing.
|
|
||||||
- Schema definitions repeat the same envelope column declarations across parent tables.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- The mapper is now the largest concentration of cross-product persistence policy. As more canonical products are added, missing an envelope column, count field, UTC conversion, or required-field check becomes easier.
|
|
||||||
- Recent SPC work showed that canonical fields can be accidentally omitted from persistence even when the model and docs are correct.
|
|
||||||
- Refactoring this area would reduce maintenance risk and make mapper tests easier to read.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Add a small `eventEnvelopeValues(e)` helper returning the parent envelope value map, then merge product-specific columns into it.
|
|
||||||
- Add a small `eventEnvelopeColumns()` helper for schema definitions if feedkit schema construction remains readable.
|
|
||||||
- Add required-field helper functions for common checks such as `requireTime`, `requireString`, and `requireJSON`, but keep product-specific validation functions where policy differs.
|
|
||||||
- Keep explicit per-product mapper functions. Do not introduce reflection-based table mapping, struct tags, or a generic ORM layer.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- Add focused helper tests for envelope value UTC/null behavior.
|
|
||||||
- Keep product mapper tests asserting important columns per product.
|
|
||||||
- Add a regression test that each parent table with event envelope columns receives all envelope values from mapper output.
|
|
||||||
- Keep schema tests for nullable/required columns, especially recent outlook and forecast condition semantics.
|
|
||||||
|
|
||||||
Risk level: Medium. The target is low-level persistence code; refactor only with existing mapper tests passing and add tests before moving column/value construction.
|
|
||||||
|
|
||||||
### 3. Centralize Event Kind And Driver Name Constants
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/sources/builtins.go`
|
|
||||||
- `internal/sources/builtins_test.go`
|
|
||||||
- `internal/sources/*/*.go`
|
|
||||||
- `internal/normalizers/*/*_test.go`
|
|
||||||
- `cmd/weatherfeeder/main_test.go`
|
|
||||||
- `cmd/weatherfeeder/config.yml`
|
|
||||||
- `examples/*.yml`
|
|
||||||
- `docs/config.md`
|
|
||||||
- `docs/internal/sources.md`
|
|
||||||
- `docs/integrations/events.md`
|
|
||||||
- `standards/schema.go`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- Schemas are centralized in `standards/schema.go`, but event kinds are repeatedly typed as string literals such as `event.Kind("forecast")`, `event.Kind("weather_story")`, and `event.Kind("outlook")`.
|
|
||||||
- Driver names are repeated in source constructors, registry entries, tests, config examples, docs, and troubleshooting text.
|
|
||||||
- The all-current-drivers test duplicates the registry table manually.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- Kinds and driver names are public operator-facing strings. A typo or stale test value can produce startup failures or documentation drift.
|
|
||||||
- The mismatch between centralized schemas and non-centralized kinds/drivers makes future feature additions more error-prone.
|
|
||||||
- The source registry already has a structured `pollDriverRegistrations` slice that can become the canonical source for driver tests.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Add event kind constants in `standards`, for example `KindObservation`, `KindForecast`, `KindForecastDiscussion`, `KindWeatherStory`, `KindAlert`, and `KindOutlook`, typed as `event.Kind` if dependency direction is acceptable. If `standards` should not import feedkit, use string constants and convert at adapter boundaries.
|
|
||||||
- Add source driver constants near source registration, for example in `internal/sources/drivers.go`, and have constructors/tests use those constants.
|
|
||||||
- Update source registry tests to derive the all-current-drivers list from `pollDriverRegistrations`, while keeping explicit negative tests for removed legacy names.
|
|
||||||
- Keep docs and YAML examples literal; they are user-facing examples and should not be generated for this cleanup pass.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- Update source registry tests to assert every registered driver builds as a `PollSource` using the registry slice.
|
|
||||||
- Add a small test that configured source `Kinds()` match the central kind constants.
|
|
||||||
- Keep example config load tests as the docs/example guardrail.
|
|
||||||
|
|
||||||
Risk level: Low. This is mostly mechanical, but care is needed to avoid import cycles if kind constants are typed with feedkit's `event.Kind`.
|
|
||||||
|
|
||||||
### 4. Centralize Config Example Coverage Around All Maintained YAML Files
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `cmd/weatherfeeder/main_test.go`
|
|
||||||
- `cmd/weatherfeeder/config.yml`
|
|
||||||
- `examples/config.minimal.yml`
|
|
||||||
- `examples/config.nats.yml`
|
|
||||||
- `examples/config.postgres.yml`
|
|
||||||
- `docs/config.md`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- The sample and copyable configs repeat driver names, event kinds, route kind lists, NWS user-agent conventions, source cadences, and sink shapes.
|
|
||||||
- `main_test.go` already verifies that `cmd/weatherfeeder/config.yml` and `examples/*.yml` load and that sources build scheduler jobs.
|
|
||||||
- There is no single test that compares example route kind lists against current source-advertised kinds or documented current kinds.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- Config examples are part of the operator contract. They tend to drift when a new canonical kind is added or renamed.
|
|
||||||
- Routes are easy to leave stale because a config can load successfully while omitting newly supported kinds from a production-oriented route example.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Keep examples explicit and copyable.
|
|
||||||
- Add tests that collect advertised source kinds from configured examples and verify route examples either intentionally match all kinds or document why they are selective.
|
|
||||||
- Add a small helper in tests for building the weatherfeeder source registry and validating all maintained configs, so config coverage stays obvious.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- Extend `TestMaintainedConfigExamplesLoad` to assert source expected kinds and scheduler jobs, which it already does, and add route-kind sanity checks if feedkit exposes compiled routes clearly enough.
|
|
||||||
- Add a docs/config example guard only if it can be kept simple; avoid parsing Markdown tables unless this repo already uses doc extraction tests.
|
|
||||||
|
|
||||||
Risk level: Low. This is test-only cleanup unless route semantics in examples are intentionally selective.
|
|
||||||
|
|
||||||
## Medium-Confidence Opportunities
|
|
||||||
|
|
||||||
### 1. Normalize Required-Time Helper Patterns Where Semantics Match
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/providers/nws/time.go`
|
|
||||||
- `internal/providers/openmeteo/time.go`
|
|
||||||
- `internal/providers/spc/time.go`
|
|
||||||
- `internal/normalizers/nws/forecast.go`
|
|
||||||
- `internal/normalizers/nws/weatherstories.go`
|
|
||||||
- `internal/normalizers/spc/convective_outlook.go`
|
|
||||||
- `internal/sources/nws/*`
|
|
||||||
- `internal/sources/spc/convective_outlook.go`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- Multiple normalizers implement required timestamp parsing with field-specific error messages.
|
|
||||||
- Multiple sources parse provider timestamps best-effort for effective-time selection.
|
|
||||||
- Providers correctly differ in timestamp formats, but callers often repeat trim/empty/UTC/error-context patterns.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- Time parsing is a domain policy hotspot. Small drift in required vs optional parsing, UTC normalization, or error wording can create subtle behavior differences.
|
|
||||||
- Required field names in errors are useful and should be preserved.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Do not force all providers through one cross-provider parser; NWS, Open-Meteo, and SPC formats differ for good reasons.
|
|
||||||
- Consider small provider-local helpers such as `ParseRequiredTime(value, field)` and `ParseOptionalTime(value)` where a provider already has a canonical parser.
|
|
||||||
- Use common helper signatures only when the failure behavior is truly identical.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- Provider helper tests for empty, malformed, and UTC-normalized timestamps.
|
|
||||||
- Normalizer tests that assert required timestamp errors include the field path.
|
|
||||||
|
|
||||||
Risk level: Medium. The duplication is real, but over-centralization could obscure provider-specific formats.
|
|
||||||
|
|
||||||
### 2. Table-Drive Source Registry Tests More Aggressively
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/sources/builtins_test.go`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- Several tests independently instantiate a registry and build one named driver.
|
|
||||||
- `TestRegisterBuiltinsRegistersAllCurrentDrivers` duplicates the same driver list that exists in `pollDriverRegistrations`.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- Adding new drivers currently requires touching both the registration table and a manually duplicated test list.
|
|
||||||
- The test suite already has the structure needed to derive cases from the registration table.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Replace individual positive registration tests with a table derived from `pollDriverRegistrations`.
|
|
||||||
- Keep one or two named tests only when they assert special policy, such as legacy driver removal.
|
|
||||||
- Keep `sourceConfigForDriver` but make it keyed off driver constants.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- One table-driven positive registration test for every driver.
|
|
||||||
- One explicit negative test for `nws_forecast` legacy driver.
|
|
||||||
|
|
||||||
Risk level: Low. This is test cleanup with minimal behavior risk.
|
|
||||||
|
|
||||||
### 3. Package-Local Fixture Helpers Are Duplicated
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `internal/providers/nws/forecast_discussion_test.go`
|
|
||||||
- `internal/providers/spc/geojson_test.go`
|
|
||||||
- `internal/sources/nws/forecast_discussion_test.go`
|
|
||||||
- `internal/sources/spc/convective_outlook_test.go`
|
|
||||||
- `internal/normalizers/nws/forecast_discussion_test.go`
|
|
||||||
- `internal/normalizers/spc/convective_outlook_test.go`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- Several tests define local helpers that read from `testdata` using `os.ReadFile` and `filepath.Join`.
|
|
||||||
- Helper names differ by package, but behavior is mostly identical.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- This is low-risk duplication, but fixture read failures and paths could be made more consistent.
|
|
||||||
- Cleaner fixture helpers would reduce noise in parser/source/normalizer tests.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Prefer package-local test helpers, not a cross-package test utility. Go package tests are easier to understand when fixtures remain near the package under test.
|
|
||||||
- Within each package with multiple test files, consolidate repeated `readTestFile` helpers into one `_test.go` helper file.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- No new behavior tests required; this cleanup is test-only.
|
|
||||||
- Run affected package tests.
|
|
||||||
|
|
||||||
Risk level: Low.
|
|
||||||
|
|
||||||
### 4. Schema, Model, And Postgres Documentation Lists Require Manual Synchronization
|
|
||||||
|
|
||||||
Affected files/packages:
|
|
||||||
|
|
||||||
- `standards/schema.go`
|
|
||||||
- `model/*.go`
|
|
||||||
- `docs/integrations/events.md`
|
|
||||||
- `docs/integrations/postgres.md`
|
|
||||||
- `docs/internal/normalizers.md`
|
|
||||||
- `docs/internal/sources.md`
|
|
||||||
- `docs/config.md`
|
|
||||||
|
|
||||||
Duplicated or near-duplicated behavior:
|
|
||||||
|
|
||||||
- Current schemas, raw mappings, canonical mappings, event kinds, and Postgres table contracts are documented in multiple current-behavior docs.
|
|
||||||
- This is partly intentional because docs serve different audiences, but all lists must be manually updated when a feature is added.
|
|
||||||
|
|
||||||
Why it matters:
|
|
||||||
|
|
||||||
- Recent feature additions touched many docs. Manual sync is workable now but will remain a recurring release risk.
|
|
||||||
- Documentation policy requires current-behavior docs to avoid speculative or stale content.
|
|
||||||
|
|
||||||
Recommended refactor:
|
|
||||||
|
|
||||||
- Do not generate docs wholesale.
|
|
||||||
- Add targeted doc consistency tests only for compact, machine-checkable facts, such as ensuring every schema constant appears in `docs/integrations/events.md` and every source driver appears in `docs/config.md`.
|
|
||||||
- Keep prose manual.
|
|
||||||
|
|
||||||
Suggested tests:
|
|
||||||
|
|
||||||
- A small standards/docs test that reads selected docs and checks for schema constants and driver constants.
|
|
||||||
- Keep examples load-tested.
|
|
||||||
|
|
||||||
Risk level: Medium. Doc tests can become brittle if they parse prose too deeply; keep them shallow.
|
|
||||||
|
|
||||||
## Boundary And Responsibility Concerns
|
|
||||||
|
|
||||||
### Source Adapter HTTP Policy Is Split Between Feedkit And Weatherfeeder
|
|
||||||
|
|
||||||
Most single-document sources rely on feedkit `HTTPSource`, while SPC implements a custom multi-document fetch loop. This boundary is acceptable because SPC's atomic bundle semantics differ from single-document polling. The concern is not the custom source itself; the concern is that shared weatherfeeder HTTP config policy is partly reimplemented in the SPC adapter.
|
|
||||||
|
|
||||||
Recommended home: keep generic HTTP mechanics in feedkit, but add a small weatherfeeder source helper for weatherfeeder-owned parameter names and validation when a source cannot use `HTTPSource` directly.
|
|
||||||
|
|
||||||
### Postgres Mapping Is Correctly Isolated But Becoming Too Dense
|
|
||||||
|
|
||||||
The Postgres mapper is in the right package and does not leak into domain or normalizer code. The package responsibility is clear. The concern is density and repeated policy, not boundary drift.
|
|
||||||
|
|
||||||
Recommended home: keep mapper helpers under `internal/sinks/postgres`. Do not move persistence concerns into `model` or normalizers.
|
|
||||||
|
|
||||||
### Event Kind Strings Lack A Canonical Home
|
|
||||||
|
|
||||||
Schemas have a clear home in `standards`; event kinds do not. Because kinds are part of routing and operator config, they deserve a comparable canonical code location.
|
|
||||||
|
|
||||||
Recommended home: `standards` is the best conceptual location if dependency direction remains clean. If importing feedkit's `event` package into `standards` is undesirable, use string constants in `standards` and convert in source adapters.
|
|
||||||
|
|
||||||
### Runtime Composition Is Appropriately Thin
|
|
||||||
|
|
||||||
`cmd/weatherfeeder/main.go` is mostly process wiring. It does not contain provider parsing or sink mapping. No refactor is recommended here beyond possibly extracting tiny helper functions if future CLI flags make startup more complex.
|
|
||||||
|
|
||||||
## Path, Key, And Naming Construction Review
|
|
||||||
|
|
||||||
Centralized enough:
|
|
||||||
|
|
||||||
- Schema strings are centralized in `standards/schema.go`.
|
|
||||||
- SPC product keys, day numbers, outlook types, and default URLs are centralized in `internal/providers/spc/product.go`.
|
|
||||||
- Postgres table names are centralized as constants in `internal/sinks/postgres/schema.go`.
|
|
||||||
- Test fixture paths are local and simple.
|
|
||||||
|
|
||||||
Needs cleanup:
|
|
||||||
|
|
||||||
- Event kind strings are repeated across source adapters, tests, YAML examples, and docs.
|
|
||||||
- Source driver names are repeated across constructors, registration, tests, docs, and examples.
|
|
||||||
- Postgres envelope column names are repeated in schema and mapper literals.
|
|
||||||
- Config route kind lists in examples are manually synchronized with supported canonical kinds.
|
|
||||||
|
|
||||||
Recommended approach:
|
|
||||||
|
|
||||||
- Add code constants for kinds and drivers first.
|
|
||||||
- Add narrow Postgres helpers for envelope column/value names second.
|
|
||||||
- Leave user-facing YAML and Markdown examples explicit, but test them against the code constants where practical.
|
|
||||||
|
|
||||||
## Resolution And Catalog Review
|
|
||||||
|
|
||||||
Current resolution model:
|
|
||||||
|
|
||||||
- Source drivers resolve through `internal/sources.RegisterBuiltins` and feedkit's source registry.
|
|
||||||
- Normalizers resolve by schema matching through `internal/normalizers.RegisterBuiltins` and feedkit's normalize processor.
|
|
||||||
- Sinks resolve through feedkit's sink registry, with weatherfeeder registering a Postgres schema mapper.
|
|
||||||
- Schemas resolve through `standards` constants.
|
|
||||||
- SPC product catalogs resolve through `internal/providers/spc` product metadata helpers.
|
|
||||||
|
|
||||||
Consistency assessment:
|
|
||||||
|
|
||||||
- Normalizer resolution is strong: schema equality is explicit and follows policy.
|
|
||||||
- Source driver resolution is explicit and readable, but test coverage duplicates driver lists rather than deriving from the registry table.
|
|
||||||
- SPC product resolution is strong and should remain provider-local.
|
|
||||||
- There is no artifact, prompt, module, profile, manifest, or object-key catalog in this repository.
|
|
||||||
|
|
||||||
Recommended centralization:
|
|
||||||
|
|
||||||
- Treat source driver constants and event kind constants as small catalogs.
|
|
||||||
- Avoid building a generic catalog framework; explicit registry tables are appropriate for this codebase.
|
|
||||||
|
|
||||||
## Config And Command-Loading Review
|
|
||||||
|
|
||||||
Current behavior:
|
|
||||||
|
|
||||||
- The executable reads exactly `config.yml` from the current working directory.
|
|
||||||
- There are no CLI flags, subcommands, profiles, environment-variable config overlays, or config path precedence rules.
|
|
||||||
- Feedkit owns top-level config loading and validation.
|
|
||||||
- Weatherfeeder source/sink constructors own driver-specific param validation.
|
|
||||||
- Maintained examples are load-tested and source-build-tested.
|
|
||||||
|
|
||||||
Consistency assessment:
|
|
||||||
|
|
||||||
- There is no duplicated command-loading behavior because there is only one command path.
|
|
||||||
- Driver-specific config validation is mostly consistent, but SPC has to duplicate some HTTP param parsing because it cannot use feedkit's single-document `HTTPSource`.
|
|
||||||
- OpenWeather's `units=metric` invariant is correctly located in `internal/providers/openweather` and enforced by the source constructor.
|
|
||||||
|
|
||||||
Intentional differences:
|
|
||||||
|
|
||||||
- SPC does not require `params.url` because it owns a fixed product catalog plus optional override maps.
|
|
||||||
- OpenWeather has stricter URL validation because unit semantics affect canonical mapping correctness.
|
|
||||||
- Single-document sources use conditional HTTP validators; SPC uses a bundle hash because it fetches multiple documents atomically.
|
|
||||||
|
|
||||||
Likely accidental drift risk:
|
|
||||||
|
|
||||||
- HTTP timeout and body-limit validation wording can differ between feedkit-backed HTTP sources and SPC.
|
|
||||||
- Future multi-document sources may copy SPC's config parsing rather than sharing a narrow helper.
|
|
||||||
|
|
||||||
## State, Manifest, Or Progress Handling Review
|
|
||||||
|
|
||||||
Current state handling:
|
|
||||||
|
|
||||||
- The daemon has no durable internal run state, manifest, checkpoint, or resume marker.
|
|
||||||
- Feedkit scheduler, dispatcher, sink fanout queues, and dedupe operate in memory.
|
|
||||||
- Single-document HTTP conditional validators are source-instance memory only.
|
|
||||||
- SPC unchanged-response behavior uses a source-local hash of the last complete bundle.
|
|
||||||
- Postgres persistence is external sink state.
|
|
||||||
|
|
||||||
Consistency assessment:
|
|
||||||
|
|
||||||
- The state model is documented and consistent with the architecture policy.
|
|
||||||
- There is no hidden filesystem state substituting for declared state.
|
|
||||||
- There is no resume/force/dry-run behavior to drift across commands.
|
|
||||||
|
|
||||||
Cleanup recommendation:
|
|
||||||
|
|
||||||
- No state/manifest refactor is needed now.
|
|
||||||
- If future durable checkpoints are added, design them explicitly rather than expanding the current in-memory dedupe or source-local hash semantics.
|
|
||||||
|
|
||||||
## Refactors To Avoid
|
|
||||||
|
|
||||||
Avoid these refactors in the next cleanup pass:
|
|
||||||
|
|
||||||
- A generic workflow engine or stage abstraction. The current runtime has source polling, normalization, dedupe, and dispatch; adding a stage framework would be speculative.
|
|
||||||
- A plugin runtime. The policy explicitly favors built-in registries over a general plugin system.
|
|
||||||
- Replacing feedkit HTTP, scheduler, dispatch, or sink infrastructure with weatherfeeder-owned equivalents.
|
|
||||||
- A generic Postgres ORM or reflection-driven mapper. The table contract is explicit and should remain readable.
|
|
||||||
- Cross-provider timestamp parsing that ignores provider-specific timestamp formats.
|
|
||||||
- Consolidating WMO mapping too aggressively. Provider-specific condition signals differ; only shared text fallback belongs in common helpers.
|
|
||||||
- Generating all docs from code. Shallow consistency tests are useful; generated manuals would fight the documentation policy's audience-specific structure.
|
|
||||||
- Collapsing all source adapters into one generic source type. The effective-time and payload policies are similar but still product-specific.
|
|
||||||
- Moving persistence tags or database column names into `model`. Canonical payloads should not depend on the Postgres sink.
|
|
||||||
|
|
||||||
## Recommended Implementation Sequence
|
|
||||||
|
|
||||||
1. Add event kind and source driver constants.
|
|
||||||
|
|
||||||
Scope: constants plus mechanical usage in source adapters, registry tests, and normalizer tests where appropriate. Keep docs/YAML literal. Run `go test ./internal/sources ./internal/normalizers/... ./cmd/weatherfeeder`.
|
|
||||||
|
|
||||||
2. Table-drive source registry tests.
|
|
||||||
|
|
||||||
Scope: derive positive source driver cases from `pollDriverRegistrations`; keep legacy-driver negative tests. Run `go test ./internal/sources ./cmd/weatherfeeder`.
|
|
||||||
|
|
||||||
3. Add source HTTP config helper for non-`HTTPSource` adapters.
|
|
||||||
|
|
||||||
Scope: centralize `user_agent`, `http_timeout`, and `http_response_body_limit_bytes` parsing for SPC and future multi-document sources. Do not alter simple `HTTPSource` adapters. Run `go test ./internal/sources/spc ./internal/sources ./cmd/weatherfeeder`.
|
|
||||||
|
|
||||||
4. Add Postgres envelope helper tests, then helper functions.
|
|
||||||
|
|
||||||
Scope: add `eventEnvelopeValues`, optionally envelope column helpers, and product-specific required-field helpers. Keep explicit mapper functions. Run `go test ./internal/sinks/postgres`.
|
|
||||||
|
|
||||||
5. Consolidate package-local fixture helpers.
|
|
||||||
|
|
||||||
Scope: per package only; no cross-package testing utility. Run affected provider/source/normalizer package tests.
|
|
||||||
|
|
||||||
6. Add shallow docs consistency tests.
|
|
||||||
|
|
||||||
Scope: verify schema constants and source driver constants appear in canonical docs. Avoid parsing Markdown tables deeply. Run `go test ./standards ./cmd/weatherfeeder` or place tests in a suitable package that can read repo docs.
|
|
||||||
|
|
||||||
7. Dead-code and legacy sweep.
|
|
||||||
|
|
||||||
Scope: after constants/tests are in place, search for obsolete schema/driver/kind literals such as removed legacy driver names. Keep explicit negative tests where they document supported removals.
|
|
||||||
|
|
||||||
## Test Strategy
|
|
||||||
|
|
||||||
Tests to add before refactoring:
|
|
||||||
|
|
||||||
- Source HTTP config helper tests for aliases, missing params, positive duration/body-limit validation, and error context.
|
|
||||||
- Postgres mapper tests that assert parent envelope columns are consistently present for every mapped canonical parent row.
|
|
||||||
- Source driver/kind constant tests if constants are introduced.
|
|
||||||
|
|
||||||
Tests to update during refactoring:
|
|
||||||
|
|
||||||
- `internal/sources/builtins_test.go` for table-driven registry coverage.
|
|
||||||
- `internal/sources/spc/convective_outlook_test.go` for shared HTTP config validation.
|
|
||||||
- `internal/sinks/postgres/map_test.go` and `schema_test.go` for envelope helper preservation.
|
|
||||||
- Existing provider/source/normalizer fixture tests if fixture helpers move.
|
|
||||||
|
|
||||||
Focused verification commands:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./cmd/weatherfeeder ./internal/sources ./internal/sources/... ./internal/providers/... ./internal/normalizers/... ./internal/sinks/postgres ./model ./standards
|
|
||||||
```
|
|
||||||
|
|
||||||
Full verification command before merging cleanup:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./...
|
|
||||||
```
|
|
||||||
|
|
||||||
## Appendix: Findings Not Worth Acting On
|
|
||||||
|
|
||||||
### Provider-Specific Source Files Share A Similar Shape
|
|
||||||
|
|
||||||
NWS, Open-Meteo, and OpenWeather source files all implement `Name`, `Kinds`, `Poll`, metadata decode, and event emission. This is acceptable because each product has distinct effective-time and metadata policy. Extract only the clearly shared HTTP/config pieces.
|
|
||||||
|
|
||||||
### Normalizer Match Methods Are Repetitive By Design
|
|
||||||
|
|
||||||
Most normalizers implement a one-line `Match` against a schema constant. This repetition is good: it keeps routing explicit and cheap. A generic schema-to-builder registry would add indirection without meaningful risk reduction.
|
|
||||||
|
|
||||||
### Provider Time Parsers Should Remain Provider-Specific
|
|
||||||
|
|
||||||
NWS, Open-Meteo, and SPC timestamp formats differ. The current provider-local parsers are easier to reason about than a broad cross-provider parser. Only required/optional wrapper patterns should be considered for cleanup.
|
|
||||||
|
|
||||||
### Documentation Repeats Some Lists Intentionally
|
|
||||||
|
|
||||||
`README.md`, `docs/config.md`, `docs/internal/sources.md`, and `docs/integrations/events.md` repeat selected feature lists for different audiences. Do not eliminate that repetition wholesale. Prefer shallow consistency tests for high-risk identifiers.
|
|
||||||
|
|
||||||
### SPC Bundle Hash State Should Stay Local
|
|
||||||
|
|
||||||
The SPC source's last-bundle hash is source-local unchanged-content state, not a general manifest/checkpoint system. Generalizing it now would be premature.
|
|
||||||
|
|
||||||
### Runtime Wiring Could Be Split Into Helpers, But Need Not Be
|
|
||||||
|
|
||||||
`cmd/weatherfeeder/main.go` is readable and policy-aligned. Extracting helper functions now would mostly move code around. Revisit only if CLI flags, config path options, metrics, or health checks are added.
|
|
||||||
@@ -1,281 +0,0 @@
|
|||||||
# Cleanup Roadmap
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
This roadmap turns the findings in `docs/roadmap/audit.md` into staged, behavior-preserving cleanup work for `weatherfeeder`.
|
|
||||||
|
|
||||||
The goal is to reduce duplication and drift risk before the next major release without changing public contracts. Implement these stages in order. Each stage should be small enough for one focused implementation prompt or commit unless the implementing agent discovers unexpected coupling.
|
|
||||||
|
|
||||||
Preserve the current architecture:
|
|
||||||
|
|
||||||
- `feedkit` remains generic daemon infrastructure.
|
|
||||||
- `weatherfeeder` keeps weather-domain policy in sources, providers, normalizers, `model`, `standards`, and Postgres mapping.
|
|
||||||
- Current-behavior docs should change only when implementation changes require them.
|
|
||||||
- Roadmap-only cleanup instructions belong here until implemented.
|
|
||||||
|
|
||||||
## Global Guardrails
|
|
||||||
|
|
||||||
- Do not change public schema strings, event kind strings, source driver names, config keys, table names, column names, column nullability, or canonical model JSON field names.
|
|
||||||
- Do not include database migrations; this is cleanup-only work.
|
|
||||||
- Do not change `feedkit` in this cleanup pass.
|
|
||||||
- Do not introduce broad framework abstractions, plugin systems, workflow engines, generic source frameworks, ORM-style mapping, reflection mapping, or persistence annotations in `model`.
|
|
||||||
- Keep `cmd/weatherfeeder` focused on runtime composition.
|
|
||||||
- Keep source fetching separate from normalizer mapping.
|
|
||||||
- Keep provider-specific timestamp parsing and WMO mapping provider-specific unless an existing common helper already has exactly matching semantics.
|
|
||||||
- Keep user-facing YAML examples and Markdown examples literal; do not generate docs.
|
|
||||||
- Run focused tests after each stage and `go test ./...` after all stages.
|
|
||||||
|
|
||||||
## Stage 1: Centralize Event Kind And Driver Constants
|
|
||||||
|
|
||||||
Add code constants for repeated weatherfeeder identifiers while preserving the literal string values.
|
|
||||||
|
|
||||||
Implementation requirements:
|
|
||||||
|
|
||||||
- Add event kind string constants in `standards`, for example:
|
|
||||||
- `KindObservation = "observation"`
|
|
||||||
- `KindForecast = "forecast"`
|
|
||||||
- `KindForecastDiscussion = "forecast_discussion"`
|
|
||||||
- `KindWeatherStory = "weather_story"`
|
|
||||||
- `KindAlert = "alert"`
|
|
||||||
- `KindOutlook = "outlook"`
|
|
||||||
- Keep kind constants typed as plain strings, not `event.Kind`, so `standards` does not import `feedkit`.
|
|
||||||
- Add source driver string constants in the provider source packages to avoid import cycles:
|
|
||||||
- NWS driver constants under `internal/sources/nws`.
|
|
||||||
- Open-Meteo driver constants under `internal/sources/openmeteo`.
|
|
||||||
- OpenWeather driver constants under `internal/sources/openweather`.
|
|
||||||
- SPC driver constant under `internal/sources/spc`.
|
|
||||||
- Update source constructors to use driver constants instead of local string literals.
|
|
||||||
- Update `Kinds()` methods and `SingleEvent` calls to use `event.Kind(standards.Kind...)`.
|
|
||||||
- Update source registration to use provider driver constants.
|
|
||||||
- Update internal tests to use constants where doing so reduces drift.
|
|
||||||
- Keep docs and YAML examples literal because they are user-facing examples.
|
|
||||||
- Do not add weather-specific kind constants to `feedkit`.
|
|
||||||
|
|
||||||
Acceptance criteria:
|
|
||||||
|
|
||||||
- All driver and event kind string values remain unchanged.
|
|
||||||
- No import cycle is introduced.
|
|
||||||
- `standards` does not import `feedkit`.
|
|
||||||
- Source constructors, emitted events, and advertised kinds behave identically.
|
|
||||||
|
|
||||||
Focused tests:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./internal/sources ./cmd/weatherfeeder
|
|
||||||
go test ./internal/normalizers/...
|
|
||||||
```
|
|
||||||
|
|
||||||
## Stage 2: Table-Drive Source Registry Tests
|
|
||||||
|
|
||||||
Reduce source registry test duplication while preserving registry coverage.
|
|
||||||
|
|
||||||
Implementation requirements:
|
|
||||||
|
|
||||||
- Replace repeated positive tests in `internal/sources/builtins_test.go` with one table-driven test derived from `pollDriverRegistrations`.
|
|
||||||
- Keep the explicit negative test proving legacy `nws_forecast` is not registered.
|
|
||||||
- Keep `sourceConfigForDriver`, but update it to use driver constants or registry-derived driver names.
|
|
||||||
- Preserve coverage that every current registered driver builds as a `PollSource`.
|
|
||||||
- Do not weaken `ValidateExpectedKinds` or scheduler job build coverage in `cmd/weatherfeeder` tests.
|
|
||||||
|
|
||||||
Acceptance criteria:
|
|
||||||
|
|
||||||
- Adding a new driver to `pollDriverRegistrations` automatically includes it in the positive registry test.
|
|
||||||
- Legacy-driver rejection remains explicitly tested.
|
|
||||||
- Test behavior remains deterministic and independent of live upstream services.
|
|
||||||
|
|
||||||
Focused tests:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./internal/sources ./cmd/weatherfeeder
|
|
||||||
```
|
|
||||||
|
|
||||||
## Stage 3: Add Shared HTTP Config Helper For Multi-Document Sources
|
|
||||||
|
|
||||||
Centralize common HTTP config parsing for sources that cannot use feedkit's single-document `HTTPSource`.
|
|
||||||
|
|
||||||
Implementation requirements:
|
|
||||||
|
|
||||||
- Add a narrow helper under `internal/sources/internal/httpconfig`.
|
|
||||||
- The helper should parse only the common source HTTP client params needed by non-`HTTPSource` sources:
|
|
||||||
- trimmed source name;
|
|
||||||
- required `params.user_agent` / `params.userAgent`;
|
|
||||||
- optional `params.http_timeout` using feedkit config duration semantics;
|
|
||||||
- optional `params.http_response_body_limit_bytes` as a positive integer.
|
|
||||||
- The helper should return values sufficient for callers to build a `transport.NewHTTPClient(timeout)` and pass a body limit to `transport.FetchBodyWithLimit`.
|
|
||||||
- Refactor only the SPC source to use this helper.
|
|
||||||
- Preserve SPC-specific config parsing in the SPC source:
|
|
||||||
- `latitude`;
|
|
||||||
- `longitude`;
|
|
||||||
- `location_id` / `locationID`;
|
|
||||||
- `location_name` / `locationName`;
|
|
||||||
- `geojson_urls`;
|
|
||||||
- `discussion_urls`;
|
|
||||||
- `rss_url` / `rssURL`.
|
|
||||||
- Do not replace `fksources.NewHTTPSource` for normal single-URL sources.
|
|
||||||
- Preserve SPC atomic bundle fetch behavior, bundle hash behavior, effective-time policy, raw payload shape, and error context.
|
|
||||||
|
|
||||||
Acceptance criteria:
|
|
||||||
|
|
||||||
- SPC constructor accepts and rejects the same configs as before.
|
|
||||||
- SPC still fetches required documents atomically and emits no partial bundle.
|
|
||||||
- Existing SPC source tests pass unchanged except for expected helper-related error wording if the wording becomes more consistent.
|
|
||||||
- No single-document source is refactored away from `fksources.NewHTTPSource`.
|
|
||||||
|
|
||||||
Focused tests:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./internal/sources/spc ./internal/sources
|
|
||||||
```
|
|
||||||
|
|
||||||
## Stage 4: Reduce Postgres Mapper Envelope Duplication
|
|
||||||
|
|
||||||
Centralize repeated Postgres parent envelope mapping without changing the table contract.
|
|
||||||
|
|
||||||
Implementation requirements:
|
|
||||||
|
|
||||||
- Add small helper functions inside `internal/sinks/postgres`.
|
|
||||||
- Centralize parent event envelope values:
|
|
||||||
- `event_id`;
|
|
||||||
- `event_kind`;
|
|
||||||
- `event_source`;
|
|
||||||
- `event_schema`;
|
|
||||||
- `event_emitted_at`;
|
|
||||||
- `event_effective_at`.
|
|
||||||
- Use the helper in every parent table mapper that stores event envelope columns.
|
|
||||||
- Optionally centralize event envelope column declarations if it keeps `schema.go` readable. If it makes the schema definition harder to scan, leave column declarations explicit.
|
|
||||||
- Keep explicit per-product mapper functions.
|
|
||||||
- Keep product-specific validation in product-specific functions where policy differs.
|
|
||||||
- Do not introduce reflection mapping, struct tags, ORM-style abstractions, generated table mapping, or persistence annotations in `model`.
|
|
||||||
- Preserve every existing table, column, nullability rule, required-field check, compact JSON behavior, UTC normalization, child positional index, and write count.
|
|
||||||
|
|
||||||
Acceptance criteria:
|
|
||||||
|
|
||||||
- Mapper output for existing valid payloads is equivalent before and after the refactor.
|
|
||||||
- Unsupported schemas still map to zero writes and no error.
|
|
||||||
- Required-field failures still include useful product/path context.
|
|
||||||
- Feedkit Postgres schema validation still receives complete rows for every declared column.
|
|
||||||
|
|
||||||
Focused tests:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./internal/sinks/postgres
|
|
||||||
```
|
|
||||||
|
|
||||||
## Stage 5: Tighten Config Example And Documentation Consistency Tests
|
|
||||||
|
|
||||||
Add shallow tests that detect identifier drift without generating or over-parsing documentation.
|
|
||||||
|
|
||||||
Implementation requirements:
|
|
||||||
|
|
||||||
- Extend maintained config tests so `cmd/weatherfeeder/config.yml` and every `examples/*.yml` remain loadable and source-buildable.
|
|
||||||
- Add shallow docs consistency tests for stable identifiers only:
|
|
||||||
- every schema constant in `standards` appears in `docs/integrations/events.md` when it is part of the current event contract;
|
|
||||||
- every registered source driver name appears in `docs/config.md`;
|
|
||||||
- every registered source driver name appears in `docs/internal/sources.md`.
|
|
||||||
- Avoid parsing Markdown tables deeply; simple file-content checks are sufficient.
|
|
||||||
- Do not generate docs.
|
|
||||||
- Do not modify current-behavior docs unless the implementation uncovers an actual stale documented identifier.
|
|
||||||
- Keep documentation policy intact: implemented behavior outside `docs/roadmap/`, future plans under `docs/roadmap/`.
|
|
||||||
|
|
||||||
Acceptance criteria:
|
|
||||||
|
|
||||||
- Identifier consistency tests fail when a new source driver or current schema is added without updating canonical docs.
|
|
||||||
- Tests are shallow and low maintenance.
|
|
||||||
- Tests do not assert prose formatting or table layout.
|
|
||||||
|
|
||||||
Focused tests:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./cmd/weatherfeeder ./standards ./internal/sources
|
|
||||||
```
|
|
||||||
|
|
||||||
## Stage 6: Consolidate Package-Local Test Fixture Helpers
|
|
||||||
|
|
||||||
Remove low-value duplicated fixture-reading code only where it is local and obvious.
|
|
||||||
|
|
||||||
Implementation requirements:
|
|
||||||
|
|
||||||
- Consolidate duplicated fixture readers only within the same Go package.
|
|
||||||
- Do not create a cross-package test utility package.
|
|
||||||
- Keep fixtures under each package's `testdata` directory.
|
|
||||||
- If a package has only one fixture helper, leave it alone.
|
|
||||||
- Do not change fixture contents unless a test already requires it.
|
|
||||||
- Do not mix this stage with parser behavior changes.
|
|
||||||
|
|
||||||
Acceptance criteria:
|
|
||||||
|
|
||||||
- Test helper duplication is reduced where multiple files in one package share the same fixture-reading behavior.
|
|
||||||
- Tests remain easy to read locally.
|
|
||||||
- No package imports a helper solely for tests from another package.
|
|
||||||
|
|
||||||
Focused tests:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./internal/providers/nws ./internal/providers/spc
|
|
||||||
go test ./internal/sources/nws ./internal/sources/spc
|
|
||||||
go test ./internal/normalizers/nws ./internal/normalizers/spc
|
|
||||||
```
|
|
||||||
|
|
||||||
## Stage 7: Dead-Code And Literal Sweep
|
|
||||||
|
|
||||||
Perform a final cleanup sweep after constants and helper stages are complete.
|
|
||||||
|
|
||||||
Implementation requirements:
|
|
||||||
|
|
||||||
- Search for stale driver, kind, and schema literals after earlier stages.
|
|
||||||
- Replace internal code/test literals with constants where it reduces typo or drift risk.
|
|
||||||
- Keep user-facing docs and YAML examples literal.
|
|
||||||
- Keep intentional legacy-driver negative tests.
|
|
||||||
- Do not remove compatibility tests unless they are clearly obsolete and no longer document supported behavior.
|
|
||||||
- Do not broaden the cleanup into unrelated refactors.
|
|
||||||
|
|
||||||
Suggested searches:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
rg 'event\.Kind\("|nws_forecast|nws_weatherstories|openmeteo_|openweather_|spc_convective_outlook|weather_story|forecast_discussion|raw\.|weather\.' .
|
|
||||||
rg 'TODO|legacy|deprecated|unknown source driver' internal cmd docs examples
|
|
||||||
```
|
|
||||||
|
|
||||||
Acceptance criteria:
|
|
||||||
|
|
||||||
- Internal literals are reduced where constants now exist.
|
|
||||||
- Intentional literals in docs, YAML examples, raw schema docs, and negative tests remain readable.
|
|
||||||
- No behavior changes are introduced.
|
|
||||||
|
|
||||||
Focused tests:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./internal/sources ./internal/normalizers/... ./internal/sinks/postgres ./cmd/weatherfeeder
|
|
||||||
```
|
|
||||||
|
|
||||||
## Final Verification
|
|
||||||
|
|
||||||
After all stages are complete, run:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./...
|
|
||||||
git status --short
|
|
||||||
```
|
|
||||||
|
|
||||||
Before committing the implemented cleanup, verify:
|
|
||||||
|
|
||||||
- No public contracts changed unintentionally.
|
|
||||||
- Current-behavior docs still describe implemented behavior only.
|
|
||||||
- `docs/roadmap/cleanup.md` is either updated to remove completed work or moved to future/remediation tracking according to the repository's roadmap practice.
|
|
||||||
- No unrelated changes are included.
|
|
||||||
|
|
||||||
## Refactors To Avoid
|
|
||||||
|
|
||||||
Do not perform these changes as part of this cleanup roadmap:
|
|
||||||
|
|
||||||
- Generic workflow or stage engine.
|
|
||||||
- Runtime plugin system.
|
|
||||||
- Weather-specific constants in `feedkit`.
|
|
||||||
- Replacing feedkit scheduler, dispatch, HTTP helpers, or sink mechanics.
|
|
||||||
- Generic source abstraction covering every source type.
|
|
||||||
- Reflection-based or generated Postgres mapper.
|
|
||||||
- ORM-style persistence layer.
|
|
||||||
- Database column metadata on canonical model structs.
|
|
||||||
- Cross-provider timestamp parser that hides provider-specific formats.
|
|
||||||
- Broad WMO mapper consolidation beyond existing common text fallback.
|
|
||||||
- Generated documentation system.
|
|
||||||
@@ -31,7 +31,7 @@ Notes:
|
|||||||
|
|
||||||
- Day 4-8 products have different semantics from Day 1-3 categorical/tornado/hail/wind products.
|
- Day 4-8 products have different semantics from Day 1-3 categorical/tornado/hail/wind products.
|
||||||
- Avoid forcing Day 4-8 assumptions into the current Day 1-3 model until the source shapes and consumer needs are reviewed.
|
- Avoid forcing Day 4-8 assumptions into the current Day 1-3 model until the source shapes and consumer needs are reviewed.
|
||||||
- Prefer reusing `weather.outlook.v1` if the fields remain accurate; otherwise write a separate roadmap before changing the canonical contract.
|
- Prefer reusing `weather.outlook.v2` if the fields remain accurate; otherwise write a separate roadmap before changing the canonical contract.
|
||||||
|
|
||||||
### Degraded SPC Bundle Mode
|
### Degraded SPC Bundle Mode
|
||||||
|
|
||||||
|
|||||||
87
docs/roadmap/outlook-schema-transition.md
Normal file
87
docs/roadmap/outlook-schema-transition.md
Normal file
@@ -0,0 +1,87 @@
|
|||||||
|
# SPC Outlook Postgres Schema Transition
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This document describes how to reset existing Postgres outlook tables from the
|
||||||
|
`weather.outlook.v1` storage shape to the `weather.outlook.v2` compatible
|
||||||
|
storage shape.
|
||||||
|
|
||||||
|
Updated `weatherfeeder` versions create outlook tables with run-level
|
||||||
|
discussion storage. Existing databases that already contain the old outlook
|
||||||
|
table family need a manual reset because the Postgres sink creates tables with
|
||||||
|
`CREATE TABLE IF NOT EXISTS`.
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
This reset drops only the outlook table family and lets updated `weatherfeeder`
|
||||||
|
recreate it:
|
||||||
|
|
||||||
|
- `outlook_discussions`
|
||||||
|
- `outlooks`
|
||||||
|
- `outlook_runs`
|
||||||
|
|
||||||
|
Other weather tables are not affected.
|
||||||
|
|
||||||
|
## Warning
|
||||||
|
|
||||||
|
These commands delete stored SPC outlook history. Existing `weather.outlook.v1`
|
||||||
|
outlook rows are intentionally removed. Downstream readers should be updated
|
||||||
|
intentionally for the new outlook shape.
|
||||||
|
|
||||||
|
## Deployment Order
|
||||||
|
|
||||||
|
1. Stop `weatherfeeder`.
|
||||||
|
2. Drop the existing outlook tables.
|
||||||
|
3. Deploy updated `weatherfeeder`.
|
||||||
|
4. Start `weatherfeeder` so the Postgres sink recreates the new outlook tables.
|
||||||
|
5. Deploy updated downstream consumers such as `weatherapi`.
|
||||||
|
|
||||||
|
## Reset SQL
|
||||||
|
|
||||||
|
```sql
|
||||||
|
DROP TABLE IF EXISTS outlook_discussions;
|
||||||
|
DROP TABLE IF EXISTS outlooks;
|
||||||
|
DROP TABLE IF EXISTS outlook_runs;
|
||||||
|
```
|
||||||
|
|
||||||
|
## Verification SQL
|
||||||
|
|
||||||
|
Before or after the updated daemon starts, this query shows which outlook tables
|
||||||
|
exist:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT table_name
|
||||||
|
FROM information_schema.tables
|
||||||
|
WHERE table_name IN ('outlook_runs', 'outlooks', 'outlook_discussions')
|
||||||
|
ORDER BY table_name;
|
||||||
|
```
|
||||||
|
|
||||||
|
After the updated daemon has started and recreated the tables, verify the new
|
||||||
|
run column:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT column_name, is_nullable, data_type
|
||||||
|
FROM information_schema.columns
|
||||||
|
WHERE table_name = 'outlook_runs'
|
||||||
|
AND column_name = 'discussion_count';
|
||||||
|
```
|
||||||
|
|
||||||
|
Verify the discussion table indexes:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT indexname
|
||||||
|
FROM pg_indexes
|
||||||
|
WHERE tablename = 'outlook_discussions'
|
||||||
|
ORDER BY indexname;
|
||||||
|
```
|
||||||
|
|
||||||
|
Verify that legacy polygon-level prose columns are gone from `outlooks`:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT column_name
|
||||||
|
FROM information_schema.columns
|
||||||
|
WHERE table_name = 'outlooks'
|
||||||
|
AND column_name IN ('headline', 'summary', 'discussion');
|
||||||
|
```
|
||||||
|
|
||||||
|
The final query should return zero rows.
|
||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/ejr/feedkit/event"
|
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestFinalizeRoundsWeatherPayloadFloats(t *testing.T) {
|
func TestFinalizeRoundsWeatherPayloadFloats(t *testing.T) {
|
||||||
@@ -14,7 +15,7 @@ func TestFinalizeRoundsWeatherPayloadFloats(t *testing.T) {
|
|||||||
|
|
||||||
in := event.Event{
|
in := event.Event{
|
||||||
ID: "evt-1",
|
ID: "evt-1",
|
||||||
Kind: event.Kind("observation"),
|
Kind: event.Kind(standards.KindObservation),
|
||||||
Source: "source-a",
|
Source: "source-a",
|
||||||
EmittedAt: time.Date(2026, 3, 28, 12, 0, 0, 0, time.UTC),
|
EmittedAt: time.Date(2026, 3, 28, 12, 0, 0, 0, time.UTC),
|
||||||
Schema: "raw.example.v1",
|
Schema: "raw.example.v1",
|
||||||
|
|||||||
@@ -18,6 +18,18 @@ import (
|
|||||||
// Errors include a small amount of operation context ("extract payload", "decode raw payload").
|
// Errors include a small amount of operation context ("extract payload", "decode raw payload").
|
||||||
// Callers typically wrap these with a provider/kind label.
|
// Callers typically wrap these with a provider/kind label.
|
||||||
func DecodeJSONPayload[T any](in event.Event) (T, error) {
|
func DecodeJSONPayload[T any](in event.Event) (T, error) {
|
||||||
|
var zero T
|
||||||
|
|
||||||
|
if typed, ok := in.Payload.(T); ok {
|
||||||
|
return typed, nil
|
||||||
|
}
|
||||||
|
if ptr, ok := in.Payload.(*T); ok {
|
||||||
|
if ptr == nil {
|
||||||
|
return zero, fmt.Errorf("extract payload: payload pointer is nil")
|
||||||
|
}
|
||||||
|
return *ptr, nil
|
||||||
|
}
|
||||||
|
|
||||||
return fknormalize.DecodeJSONPayload[T](in)
|
return fknormalize.DecodeJSONPayload[T](in)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
39
internal/normalizers/common/json_test.go
Normal file
39
internal/normalizers/common/json_test.go
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
package common
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDecodeJSONPayloadAcceptsTypedPayload(t *testing.T) {
|
||||||
|
type rawPayload struct {
|
||||||
|
Value string `json:"value"`
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := DecodeJSONPayload[rawPayload](event.Event{
|
||||||
|
Payload: rawPayload{Value: "ok"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("DecodeJSONPayload() error = %v", err)
|
||||||
|
}
|
||||||
|
if got.Value != "ok" {
|
||||||
|
t.Fatalf("Value = %q, want ok", got.Value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDecodeJSONPayloadAcceptsTypedPointerPayload(t *testing.T) {
|
||||||
|
type rawPayload struct {
|
||||||
|
Value string `json:"value"`
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := DecodeJSONPayload[rawPayload](event.Event{
|
||||||
|
Payload: &rawPayload{Value: "ok"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("DecodeJSONPayload() error = %v", err)
|
||||||
|
}
|
||||||
|
if got.Value != "ok" {
|
||||||
|
t.Fatalf("Value = %q, want ok", got.Value)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -18,7 +18,7 @@ func TestForecastDiscussionNormalizerProducesCanonicalSchema(t *testing.T) {
|
|||||||
|
|
||||||
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
|
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
|
||||||
ID: "evt-discussion-1",
|
ID: "evt-discussion-1",
|
||||||
Kind: event.Kind("forecast_discussion"),
|
Kind: event.Kind(standards.KindForecastDiscussion),
|
||||||
Source: "nws-discussion-test",
|
Source: "nws-discussion-test",
|
||||||
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
|
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
|
||||||
Schema: standards.SchemaRawNWSForecastDiscussionV1,
|
Schema: standards.SchemaRawNWSForecastDiscussionV1,
|
||||||
@@ -33,7 +33,7 @@ func TestForecastDiscussionNormalizerProducesCanonicalSchema(t *testing.T) {
|
|||||||
if out.Schema != standards.SchemaWeatherForecastDiscussionV1 {
|
if out.Schema != standards.SchemaWeatherForecastDiscussionV1 {
|
||||||
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherForecastDiscussionV1)
|
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherForecastDiscussionV1)
|
||||||
}
|
}
|
||||||
if out.Kind != event.Kind("forecast_discussion") {
|
if out.Kind != event.Kind(standards.KindForecastDiscussion) {
|
||||||
t.Fatalf("Kind = %q, want forecast_discussion", out.Kind)
|
t.Fatalf("Kind = %q, want forecast_discussion", out.Kind)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -74,7 +74,7 @@ func TestForecastDiscussionNormalizerProducesCanonicalSchema(t *testing.T) {
|
|||||||
func TestForecastDiscussionNormalizerRejectsMissingIssueTime(t *testing.T) {
|
func TestForecastDiscussionNormalizerRejectsMissingIssueTime(t *testing.T) {
|
||||||
_, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
|
_, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
|
||||||
ID: "evt-discussion-bad",
|
ID: "evt-discussion-bad",
|
||||||
Kind: event.Kind("forecast_discussion"),
|
Kind: event.Kind(standards.KindForecastDiscussion),
|
||||||
Source: "nws-discussion-test",
|
Source: "nws-discussion-test",
|
||||||
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
|
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
|
||||||
Schema: standards.SchemaRawNWSForecastDiscussionV1,
|
Schema: standards.SchemaRawNWSForecastDiscussionV1,
|
||||||
@@ -93,7 +93,7 @@ func TestForecastDiscussionNormalizerWireShapeHasNoUnexpectedKeys(t *testing.T)
|
|||||||
|
|
||||||
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
|
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
|
||||||
ID: "evt-discussion-2",
|
ID: "evt-discussion-2",
|
||||||
Kind: event.Kind("forecast_discussion"),
|
Kind: event.Kind(standards.KindForecastDiscussion),
|
||||||
Source: "nws-discussion-test",
|
Source: "nws-discussion-test",
|
||||||
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
|
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
|
||||||
Schema: standards.SchemaRawNWSForecastDiscussionV1,
|
Schema: standards.SchemaRawNWSForecastDiscussionV1,
|
||||||
|
|||||||
@@ -182,7 +182,7 @@ func TestNormalizeForecastEventBySchemaProducesCanonicalWeatherForecastSchema(t
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
out, err := normalizeForecastEventBySchema(event.Event{
|
out, err := normalizeForecastEventBySchema(event.Event{
|
||||||
ID: "evt-1",
|
ID: "evt-1",
|
||||||
Kind: event.Kind("forecast"),
|
Kind: event.Kind(standards.KindForecast),
|
||||||
Source: "nws-test",
|
Source: "nws-test",
|
||||||
EmittedAt: time.Date(2026, 3, 16, 18, 0, 0, 0, time.UTC),
|
EmittedAt: time.Date(2026, 3, 16, 18, 0, 0, 0, time.UTC),
|
||||||
Schema: tt.schema,
|
Schema: tt.schema,
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ func TestWeatherStoriesNormalizerProducesCanonicalSchemaAndMapsSample(t *testing
|
|||||||
if out.Schema != standards.SchemaWeatherStoryV1 {
|
if out.Schema != standards.SchemaWeatherStoryV1 {
|
||||||
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherStoryV1)
|
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherStoryV1)
|
||||||
}
|
}
|
||||||
if out.Kind != event.Kind("weather_story") {
|
if out.Kind != event.Kind(standards.KindWeatherStory) {
|
||||||
t.Fatalf("Kind = %q, want weather_story", out.Kind)
|
t.Fatalf("Kind = %q, want weather_story", out.Kind)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -118,7 +118,7 @@ func TestWeatherStoriesNormalizerMatch(t *testing.T) {
|
|||||||
func weatherStoriesRawEvent(payload string) event.Event {
|
func weatherStoriesRawEvent(payload string) event.Event {
|
||||||
return event.Event{
|
return event.Event{
|
||||||
ID: "evt-weatherstories-1",
|
ID: "evt-weatherstories-1",
|
||||||
Kind: event.Kind("weather_story"),
|
Kind: event.Kind(standards.KindWeatherStory),
|
||||||
Source: "nws-weatherstories-test",
|
Source: "nws-weatherstories-test",
|
||||||
EmittedAt: time.Date(2026, 5, 30, 9, 5, 0, 0, time.UTC),
|
EmittedAt: time.Date(2026, 5, 30, 9, 5, 0, 0, time.UTC),
|
||||||
Schema: standards.SchemaRawNWSWeatherStoriesV1,
|
Schema: standards.SchemaRawNWSWeatherStoriesV1,
|
||||||
|
|||||||
@@ -22,7 +22,6 @@ const (
|
|||||||
providerSPC = "spc"
|
providerSPC = "spc"
|
||||||
productConvective = "convective"
|
productConvective = "convective"
|
||||||
outlookNormalizer = "spc convective outlook"
|
outlookNormalizer = "spc convective outlook"
|
||||||
outlookKind = "outlook"
|
|
||||||
outlookTypeUnknown = 99
|
outlookTypeUnknown = 99
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -30,10 +29,10 @@ var idTokenRE = regexp.MustCompile(`[^a-z0-9]+`)
|
|||||||
|
|
||||||
// ConvectiveOutlookNormalizer converts:
|
// ConvectiveOutlookNormalizer converts:
|
||||||
//
|
//
|
||||||
// standards.SchemaRawSPCConvectiveOutlookV1 -> standards.SchemaWeatherOutlookV1
|
// standards.SchemaRawSPCConvectiveOutlookV1 -> standards.SchemaWeatherOutlookV2
|
||||||
//
|
//
|
||||||
// It maps SPC GeoJSON outlook features into canonical outlook polygons and
|
// It maps SPC GeoJSON outlook features containing the configured location into
|
||||||
// enriches each day with the matching required print-page discussion.
|
// canonical outlook polygons and adds matching day-level print-page discussions.
|
||||||
type ConvectiveOutlookNormalizer struct{}
|
type ConvectiveOutlookNormalizer struct{}
|
||||||
|
|
||||||
func (ConvectiveOutlookNormalizer) Match(e event.Event) bool {
|
func (ConvectiveOutlookNormalizer) Match(e event.Event) bool {
|
||||||
@@ -51,7 +50,7 @@ func (ConvectiveOutlookNormalizer) Normalize(ctx context.Context, in event.Event
|
|||||||
return normcommon.NormalizeJSON(
|
return normcommon.NormalizeJSON(
|
||||||
in,
|
in,
|
||||||
outlookNormalizer,
|
outlookNormalizer,
|
||||||
standards.SchemaWeatherOutlookV1,
|
standards.SchemaWeatherOutlookV2,
|
||||||
func(parsed spcprovider.RawConvectiveOutlookBundle) (model.WeatherOutlookRun, time.Time, error) {
|
func(parsed spcprovider.RawConvectiveOutlookBundle) (model.WeatherOutlookRun, time.Time, error) {
|
||||||
return buildConvectiveOutlook(parsed, fallbackAsOf)
|
return buildConvectiveOutlook(parsed, fallbackAsOf)
|
||||||
},
|
},
|
||||||
@@ -77,8 +76,7 @@ func buildConvectiveOutlook(bundle spcprovider.RawConvectiveOutlookBundle, fallb
|
|||||||
if err := validateProductMetadata(product); err != nil {
|
if err := validateProductMetadata(product); err != nil {
|
||||||
return model.WeatherOutlookRun{}, time.Time{}, err
|
return model.WeatherOutlookRun{}, time.Time{}, err
|
||||||
}
|
}
|
||||||
discussion, ok := discussions[product.Day]
|
if _, ok := discussions[product.Day]; !ok {
|
||||||
if !ok {
|
|
||||||
return model.WeatherOutlookRun{}, time.Time{}, fmt.Errorf("product %s: discussion for day %d is required", product.Key, product.Day)
|
return model.WeatherOutlookRun{}, time.Time{}, fmt.Errorf("product %s: discussion for day %d is required", product.Key, product.Day)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -88,17 +86,36 @@ func buildConvectiveOutlook(bundle spcprovider.RawConvectiveOutlookBundle, fallb
|
|||||||
}
|
}
|
||||||
|
|
||||||
for i, feature := range collection.Features {
|
for i, feature := range collection.Features {
|
||||||
outlook, err := mapFeature(product, feature, i, point, discussion)
|
if spcprovider.IsEmptyGeometryCollection(feature.Geometry) {
|
||||||
|
issuedAt, err := parseRequiredSPCTime(feature.Properties.IssueISO, fmt.Sprintf("product %s feature %d.ISSUE_ISO", product.Key, i))
|
||||||
|
if err != nil {
|
||||||
|
return model.WeatherOutlookRun{}, time.Time{}, err
|
||||||
|
}
|
||||||
|
if latestIssue.IsZero() || issuedAt.After(latestIssue) {
|
||||||
|
latestIssue = issuedAt
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
outlook, err := mapFeature(product, feature, i, point)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return model.WeatherOutlookRun{}, time.Time{}, err
|
return model.WeatherOutlookRun{}, time.Time{}, err
|
||||||
}
|
}
|
||||||
if latestIssue.IsZero() || outlook.IssuedAt.After(latestIssue) {
|
if latestIssue.IsZero() || outlook.IssuedAt.After(latestIssue) {
|
||||||
latestIssue = outlook.IssuedAt
|
latestIssue = outlook.IssuedAt
|
||||||
}
|
}
|
||||||
|
if !outlook.ContainsLocation {
|
||||||
|
continue
|
||||||
|
}
|
||||||
outlooks = append(outlooks, outlook)
|
outlooks = append(outlooks, outlook)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
runDiscussions, err := buildOutlookDiscussions(outlooks, discussions)
|
||||||
|
if err != nil {
|
||||||
|
return model.WeatherOutlookRun{}, time.Time{}, err
|
||||||
|
}
|
||||||
|
|
||||||
asOf := latestIssue
|
asOf := latestIssue
|
||||||
if asOf.IsZero() {
|
if asOf.IsZero() {
|
||||||
asOf = latestDiscussionUpdated
|
asOf = latestDiscussionUpdated
|
||||||
@@ -123,6 +140,7 @@ func buildConvectiveOutlook(bundle spcprovider.RawConvectiveOutlookBundle, fallb
|
|||||||
AsOf: asOf.UTC(),
|
AsOf: asOf.UTC(),
|
||||||
IssuedAt: issuedAt,
|
IssuedAt: issuedAt,
|
||||||
Outlooks: outlooks,
|
Outlooks: outlooks,
|
||||||
|
Discussions: runDiscussions,
|
||||||
}
|
}
|
||||||
return run, run.AsOf, nil
|
return run, run.AsOf, nil
|
||||||
}
|
}
|
||||||
@@ -165,6 +183,43 @@ func parseDiscussions(pages []spcprovider.RawDiscussionPage) (map[int]parsedDisc
|
|||||||
return out, latestUpdated, nil
|
return out, latestUpdated, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func buildOutlookDiscussions(outlooks []model.WeatherOutlook, discussions map[int]parsedDiscussion) ([]model.WeatherOutlookDiscussion, error) {
|
||||||
|
daysWithOutlooks := map[int]bool{}
|
||||||
|
for _, outlook := range outlooks {
|
||||||
|
daysWithOutlooks[outlook.Day] = true
|
||||||
|
}
|
||||||
|
|
||||||
|
days := make([]int, 0, len(daysWithOutlooks))
|
||||||
|
for day := range daysWithOutlooks {
|
||||||
|
days = append(days, day)
|
||||||
|
}
|
||||||
|
sort.Ints(days)
|
||||||
|
|
||||||
|
out := make([]model.WeatherOutlookDiscussion, 0, len(days))
|
||||||
|
for _, day := range days {
|
||||||
|
disc, ok := discussions[day]
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("discussion for retained day %d is required", day)
|
||||||
|
}
|
||||||
|
out = append(out, model.WeatherOutlookDiscussion{
|
||||||
|
Day: day,
|
||||||
|
Headline: disc.Headline,
|
||||||
|
Summary: disc.Summary,
|
||||||
|
Discussion: disc.Discussion,
|
||||||
|
UpdatedAt: utcTimePtr(disc.UpdatedAt),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func utcTimePtr(t *time.Time) *time.Time {
|
||||||
|
if t == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
tt := t.UTC()
|
||||||
|
return &tt
|
||||||
|
}
|
||||||
|
|
||||||
func orderedProducts(products []spcprovider.RawOutlookProduct) []spcprovider.RawOutlookProduct {
|
func orderedProducts(products []spcprovider.RawOutlookProduct) []spcprovider.RawOutlookProduct {
|
||||||
out := make([]spcprovider.RawOutlookProduct, len(products))
|
out := make([]spcprovider.RawOutlookProduct, len(products))
|
||||||
copy(out, products)
|
copy(out, products)
|
||||||
@@ -209,7 +264,7 @@ func validateProductMetadata(product spcprovider.RawOutlookProduct) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func mapFeature(product spcprovider.RawOutlookProduct, feature spcprovider.GeoJSONFeature, index int, point geo.Point, discussion parsedDiscussion) (model.WeatherOutlook, error) {
|
func mapFeature(product spcprovider.RawOutlookProduct, feature spcprovider.GeoJSONFeature, index int, point geo.Point) (model.WeatherOutlook, error) {
|
||||||
fieldPrefix := fmt.Sprintf("product %s feature %d", product.Key, index)
|
fieldPrefix := fmt.Sprintf("product %s feature %d", product.Key, index)
|
||||||
props := feature.Properties
|
props := feature.Properties
|
||||||
|
|
||||||
@@ -254,9 +309,6 @@ func mapFeature(product spcprovider.RawOutlookProduct, feature spcprovider.GeoJS
|
|||||||
IssuedAt: issuedAt,
|
IssuedAt: issuedAt,
|
||||||
ExpiresAt: validTo,
|
ExpiresAt: validTo,
|
||||||
Forecaster: strings.TrimSpace(props.Forecaster),
|
Forecaster: strings.TrimSpace(props.Forecaster),
|
||||||
Headline: discussion.Headline,
|
|
||||||
Summary: discussion.Summary,
|
|
||||||
Discussion: discussion.Discussion,
|
|
||||||
SourceURL: strings.TrimSpace(product.URL),
|
SourceURL: strings.TrimSpace(product.URL),
|
||||||
ImageURL: "",
|
ImageURL: "",
|
||||||
ContainsLocation: containsLocation,
|
ContainsLocation: containsLocation,
|
||||||
|
|||||||
@@ -2,8 +2,6 @@ package spc
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -29,10 +27,10 @@ func TestConvectiveOutlookNormalizerProducesCanonicalSchemaAndMapsSample(t *test
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Normalize() error = %v", err)
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
}
|
}
|
||||||
if out.Schema != standards.SchemaWeatherOutlookV1 {
|
if out.Schema != standards.SchemaWeatherOutlookV2 {
|
||||||
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherOutlookV1)
|
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherOutlookV2)
|
||||||
}
|
}
|
||||||
if out.Kind != event.Kind("outlook") {
|
if out.Kind != event.Kind(standards.KindOutlook) {
|
||||||
t.Fatalf("Kind = %q, want outlook", out.Kind)
|
t.Fatalf("Kind = %q, want outlook", out.Kind)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -56,8 +54,23 @@ func TestConvectiveOutlookNormalizerProducesCanonicalSchemaAndMapsSample(t *test
|
|||||||
if run.Latitude == nil || *run.Latitude != 38.5 || run.Longitude == nil || *run.Longitude != -90.5 {
|
if run.Latitude == nil || *run.Latitude != 38.5 || run.Longitude == nil || *run.Longitude != -90.5 {
|
||||||
t.Fatalf("coordinates = %v,%v", run.Latitude, run.Longitude)
|
t.Fatalf("coordinates = %v,%v", run.Latitude, run.Longitude)
|
||||||
}
|
}
|
||||||
if len(run.Outlooks) != 12 {
|
if len(run.Outlooks) != 4 {
|
||||||
t.Fatalf("Outlooks length = %d, want 12", len(run.Outlooks))
|
t.Fatalf("Outlooks length = %d, want 4", len(run.Outlooks))
|
||||||
|
}
|
||||||
|
assertAllOutlooksContainLocation(t, run.Outlooks)
|
||||||
|
assertDiscussionDays(t, run.Discussions, 1)
|
||||||
|
day1Discussion := run.Discussions[0]
|
||||||
|
if day1Discussion.Headline != "Day 1 Convective Outlook" {
|
||||||
|
t.Fatalf("day 1 Headline = %q", day1Discussion.Headline)
|
||||||
|
}
|
||||||
|
if !strings.Contains(day1Discussion.Summary, "central Plains") {
|
||||||
|
t.Fatalf("day 1 Summary = %q", day1Discussion.Summary)
|
||||||
|
}
|
||||||
|
if !strings.Contains(day1Discussion.Discussion, "...DISCUSSION...") {
|
||||||
|
t.Fatalf("day 1 Discussion missing product text: %q", day1Discussion.Discussion)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(day1Discussion.Discussion, "SPC AC 111234") {
|
||||||
|
t.Fatalf("day 1 Discussion = %q, want SPC product code prefix", day1Discussion.Discussion)
|
||||||
}
|
}
|
||||||
|
|
||||||
got := run.Outlooks[0]
|
got := run.Outlooks[0]
|
||||||
@@ -90,25 +103,54 @@ func TestConvectiveOutlookNormalizerProducesCanonicalSchemaAndMapsSample(t *test
|
|||||||
if !got.ContainsLocation {
|
if !got.ContainsLocation {
|
||||||
t.Fatalf("ContainsLocation = false, want true")
|
t.Fatalf("ContainsLocation = false, want true")
|
||||||
}
|
}
|
||||||
if got.Headline != "Day 1 Convective Outlook" {
|
|
||||||
t.Fatalf("Headline = %q", got.Headline)
|
|
||||||
}
|
|
||||||
if !strings.Contains(got.Summary, "central Plains") {
|
|
||||||
t.Fatalf("Summary = %q", got.Summary)
|
|
||||||
}
|
|
||||||
if !strings.Contains(got.Discussion, "...DISCUSSION...") {
|
|
||||||
t.Fatalf("Discussion missing product text: %q", got.Discussion)
|
|
||||||
}
|
|
||||||
if !strings.HasPrefix(got.Discussion, "SPC AC 111234") {
|
|
||||||
t.Fatalf("Discussion = %q, want SPC product code prefix", got.Discussion)
|
|
||||||
}
|
|
||||||
if got.ID != "spc-convective-day1-categorical-slgt-2026-06-11T12:34:56Z-2026-06-11T13:00:00Z-0" {
|
if got.ID != "spc-convective-day1-categorical-slgt-2026-06-11T12:34:56Z-2026-06-11T13:00:00Z-0" {
|
||||||
t.Fatalf("ID = %q", got.ID)
|
t.Fatalf("ID = %q", got.ID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerAcceptsTypedSourcePayload(t *testing.T) {
|
||||||
|
bundle := spcBundle(t, 38.5, -90.5)
|
||||||
|
in := spcRawEvent(t, bundle)
|
||||||
|
in.Payload = bundle
|
||||||
|
|
||||||
|
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, in)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
run := out.Payload.(model.WeatherOutlookRun)
|
||||||
|
if len(run.Outlooks) != 4 {
|
||||||
|
t.Fatalf("Outlooks length = %d, want 4", len(run.Outlooks))
|
||||||
|
}
|
||||||
|
assertAllOutlooksContainLocation(t, run.Outlooks)
|
||||||
|
assertDiscussionDays(t, run.Discussions, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerEmitsEmptyLocalRunOutsidePolygons(t *testing.T) {
|
||||||
|
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 0, 0)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
run := out.Payload.(model.WeatherOutlookRun)
|
||||||
|
if len(run.Outlooks) != 0 {
|
||||||
|
t.Fatalf("Outlooks length = %d, want 0", len(run.Outlooks))
|
||||||
|
}
|
||||||
|
if len(run.Discussions) != 0 {
|
||||||
|
t.Fatalf("Discussions length = %d, want 0", len(run.Discussions))
|
||||||
|
}
|
||||||
|
wantAsOf := time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)
|
||||||
|
if !run.AsOf.Equal(wantAsOf) {
|
||||||
|
t.Fatalf("AsOf = %s, want latest product issue time %s", run.AsOf, wantAsOf)
|
||||||
|
}
|
||||||
|
if run.IssuedAt == nil || !run.IssuedAt.Equal(wantAsOf) {
|
||||||
|
t.Fatalf("IssuedAt = %v, want latest product issue time %s", run.IssuedAt, wantAsOf)
|
||||||
|
}
|
||||||
|
if out.EffectiveAt == nil || !out.EffectiveAt.Equal(run.AsOf) {
|
||||||
|
t.Fatalf("EffectiveAt = %v, want run AsOf %s", out.EffectiveAt, run.AsOf)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestConvectiveOutlookNormalizerOrdersProductsByDayAndType(t *testing.T) {
|
func TestConvectiveOutlookNormalizerOrdersProductsByDayAndType(t *testing.T) {
|
||||||
bundle := spcBundle(t, 0, 0)
|
bundle := spcBundle(t, 38.5, -90.5)
|
||||||
for i, j := 0, len(bundle.Products)-1; i < j; i, j = i+1, j-1 {
|
for i, j := 0, len(bundle.Products)-1; i < j; i, j = i+1, j-1 {
|
||||||
bundle.Products[i], bundle.Products[j] = bundle.Products[j], bundle.Products[i]
|
bundle.Products[i], bundle.Products[j] = bundle.Products[j], bundle.Products[i]
|
||||||
}
|
}
|
||||||
@@ -138,11 +180,12 @@ func TestConvectiveOutlookNormalizerOrdersProductsByDayAndType(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestConvectiveOutlookNormalizerMapsProbabilisticOutlookTypes(t *testing.T) {
|
func TestConvectiveOutlookNormalizerMapsProbabilisticOutlookTypes(t *testing.T) {
|
||||||
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 0, 0)))
|
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 38.5, -90.5)))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Normalize() error = %v", err)
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
}
|
}
|
||||||
run := out.Payload.(model.WeatherOutlookRun)
|
run := out.Payload.(model.WeatherOutlookRun)
|
||||||
|
assertAllOutlooksContainLocation(t, run.Outlooks)
|
||||||
for _, outlookType := range []string{
|
for _, outlookType := range []string{
|
||||||
spcprovider.OutlookTypeTornado,
|
spcprovider.OutlookTypeTornado,
|
||||||
spcprovider.OutlookTypeHail,
|
spcprovider.OutlookTypeHail,
|
||||||
@@ -154,19 +197,75 @@ func TestConvectiveOutlookNormalizerMapsProbabilisticOutlookTypes(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestConvectiveOutlookNormalizerContainsLocationFalseOutsidePolygon(t *testing.T) {
|
func TestConvectiveOutlookNormalizerSkipsEmptyGeometryCollectionPlaceholder(t *testing.T) {
|
||||||
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 0, 0)))
|
bundle := spcBundle(t, 36, -99)
|
||||||
|
replaced := false
|
||||||
|
for i := range bundle.Products {
|
||||||
|
if bundle.Products[i].Day == 2 && bundle.Products[i].OutlookType == spcprovider.OutlookTypeTornado {
|
||||||
|
bundle.Products[i].Body = json.RawMessage(emptyGeometryCollectionGeoJSON())
|
||||||
|
replaced = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !replaced {
|
||||||
|
t.Fatalf("test setup did not find day 2 tornado product")
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Normalize() error = %v", err)
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
}
|
}
|
||||||
run := out.Payload.(model.WeatherOutlookRun)
|
run := out.Payload.(model.WeatherOutlookRun)
|
||||||
if run.Outlooks[0].ContainsLocation {
|
if len(run.Outlooks) != 3 {
|
||||||
t.Fatalf("ContainsLocation = true, want false")
|
t.Fatalf("Outlooks length = %d, want 3", len(run.Outlooks))
|
||||||
|
}
|
||||||
|
assertAllOutlooksContainLocation(t, run.Outlooks)
|
||||||
|
assertDiscussionDays(t, run.Discussions, 2)
|
||||||
|
if got := findOutlook(run.Outlooks, 2, spcprovider.OutlookTypeTornado); got != nil {
|
||||||
|
t.Fatalf("day 2 tornado outlook = %+v, want nil placeholder skipped", *got)
|
||||||
|
}
|
||||||
|
wantAsOf := time.Date(2026, 6, 12, 10, 0, 0, 0, time.UTC)
|
||||||
|
if !run.AsOf.Equal(wantAsOf) {
|
||||||
|
t.Fatalf("AsOf = %s, want placeholder ISSUE_ISO %s", run.AsOf, wantAsOf)
|
||||||
|
}
|
||||||
|
if out.EffectiveAt == nil || !out.EffectiveAt.Equal(wantAsOf) {
|
||||||
|
t.Fatalf("EffectiveAt = %v, want placeholder ISSUE_ISO %s", out.EffectiveAt, wantAsOf)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerIncludesOnlyDayWithContainingPolygons(t *testing.T) {
|
||||||
|
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 36, -99)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
run := out.Payload.(model.WeatherOutlookRun)
|
||||||
|
if len(run.Outlooks) == 0 {
|
||||||
|
t.Fatalf("Outlooks length = 0, want retained day 2 outlooks")
|
||||||
|
}
|
||||||
|
assertAllOutlooksContainLocation(t, run.Outlooks)
|
||||||
|
for i, outlook := range run.Outlooks {
|
||||||
|
if outlook.Day != 2 {
|
||||||
|
t.Fatalf("outlook[%d].Day = %d, want 2", i, outlook.Day)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assertDiscussionDays(t, run.Discussions, 2)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerUsesOneDiscussionForMultipleSameDayOutlooks(t *testing.T) {
|
||||||
|
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 38.5, -90.5)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
|
}
|
||||||
|
run := out.Payload.(model.WeatherOutlookRun)
|
||||||
|
if got := countOutlooksByDay(run.Outlooks, 1); got < 2 {
|
||||||
|
t.Fatalf("day 1 outlook count = %d, want multiple", got)
|
||||||
|
}
|
||||||
|
assertAllOutlooksContainLocation(t, run.Outlooks)
|
||||||
|
assertDiscussionDays(t, run.Discussions, 1)
|
||||||
|
}
|
||||||
|
|
||||||
func TestConvectiveOutlookNormalizerPreservesCorrectionMarker(t *testing.T) {
|
func TestConvectiveOutlookNormalizerPreservesCorrectionMarker(t *testing.T) {
|
||||||
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 0, 0)))
|
bundle := spcBundle(t, 36, -99)
|
||||||
|
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Normalize() error = %v", err)
|
t.Fatalf("Normalize() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -175,11 +274,12 @@ func TestConvectiveOutlookNormalizerPreservesCorrectionMarker(t *testing.T) {
|
|||||||
if got == nil {
|
if got == nil {
|
||||||
t.Fatalf("missing day 2 tornado outlook")
|
t.Fatalf("missing day 2 tornado outlook")
|
||||||
}
|
}
|
||||||
if !strings.Contains(got.Headline, "CORR 1") {
|
assertDiscussionDays(t, run.Discussions, 2)
|
||||||
t.Fatalf("Headline = %q, want correction marker", got.Headline)
|
if !strings.Contains(run.Discussions[0].Headline, "CORR 1") {
|
||||||
|
t.Fatalf("day 2 headline = %q, want correction marker", run.Discussions[0].Headline)
|
||||||
}
|
}
|
||||||
if !strings.Contains(got.Discussion, "CORR 1") {
|
if !strings.Contains(run.Discussions[0].Discussion, "CORR 1") {
|
||||||
t.Fatalf("Discussion = %q, want correction marker", got.Discussion)
|
t.Fatalf("day 2 discussion = %q, want correction marker", run.Discussions[0].Discussion)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -245,6 +345,19 @@ func TestConvectiveOutlookNormalizerRejectsMissingLabel(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestConvectiveOutlookNormalizerRejectsMissingDiscussion(t *testing.T) {
|
||||||
|
bundle := spcBundle(t, 38.5, -90.5)
|
||||||
|
bundle.Discussions = bundle.Discussions[1:]
|
||||||
|
|
||||||
|
_, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, bundle))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("Normalize() error = nil, want error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "product day1_categorical: discussion for day 1 is required") {
|
||||||
|
t.Fatalf("error = %q, want missing discussion context", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestConvectiveOutlookNormalizerOutputJSONShape(t *testing.T) {
|
func TestConvectiveOutlookNormalizerOutputJSONShape(t *testing.T) {
|
||||||
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 38.5, -90.5)))
|
out, err := (ConvectiveOutlookNormalizer{}).Normalize(nil, spcRawEvent(t, spcBundle(t, 38.5, -90.5)))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -255,18 +368,53 @@ func TestConvectiveOutlookNormalizerOutputJSONShape(t *testing.T) {
|
|||||||
t.Fatalf("Marshal(payload) error = %v", err)
|
t.Fatalf("Marshal(payload) error = %v", err)
|
||||||
}
|
}
|
||||||
got := string(raw)
|
got := string(raw)
|
||||||
for _, want := range []string{`"asOf"`, `"outlooks"`, `"containsLocation"`, `"geometry"`} {
|
for _, want := range []string{`"asOf"`, `"outlooks"`, `"discussions"`, `"containsLocation"`, `"geometry"`} {
|
||||||
if !strings.Contains(got, want) {
|
if !strings.Contains(got, want) {
|
||||||
t.Fatalf("payload JSON missing %s: %s", want, got)
|
t.Fatalf("payload JSON missing %s: %s", want, got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, unwanted := range []string{`"products"`, `"discussions"`, `"fetchedAt"`, `"body"`} {
|
outlookStart := strings.Index(got, `"outlooks"`)
|
||||||
|
discussionStart := strings.Index(got, `"discussions"`)
|
||||||
|
if outlookStart == -1 || discussionStart == -1 || discussionStart <= outlookStart {
|
||||||
|
t.Fatalf("payload JSON has unexpected outlook/discussion order: %s", got)
|
||||||
|
}
|
||||||
|
outlookJSON := got[outlookStart:discussionStart]
|
||||||
|
for _, unwanted := range []string{`"headline"`, `"summary"`, `"discussion"`} {
|
||||||
|
if strings.Contains(outlookJSON, unwanted) {
|
||||||
|
t.Fatalf("outlook JSON exposed polygon-level prose key %s: %s", unwanted, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, unwanted := range []string{`"products"`, `"fetchedAt"`, `"body"`} {
|
||||||
if strings.Contains(got, unwanted) {
|
if strings.Contains(got, unwanted) {
|
||||||
t.Fatalf("payload JSON exposed raw key %s: %s", unwanted, got)
|
t.Fatalf("payload JSON exposed raw key %s: %s", unwanted, got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func emptyGeometryCollectionGeoJSON() []byte {
|
||||||
|
return []byte(`{
|
||||||
|
"type": "FeatureCollection",
|
||||||
|
"features": [
|
||||||
|
{
|
||||||
|
"type": "Feature",
|
||||||
|
"properties": {
|
||||||
|
"VALID_ISO": "2026-06-12T12:00:00Z",
|
||||||
|
"EXPIRE_ISO": "2026-06-13T12:00:00Z",
|
||||||
|
"ISSUE_ISO": "2026-06-12T10:00:00Z",
|
||||||
|
"FORECASTER": "DOE",
|
||||||
|
"LABEL": "Less Than 2% All Areas",
|
||||||
|
"LABEL2": "",
|
||||||
|
"DN": 0
|
||||||
|
},
|
||||||
|
"geometry": {
|
||||||
|
"type": "GeometryCollection",
|
||||||
|
"geometries": []
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}`)
|
||||||
|
}
|
||||||
|
|
||||||
func spcRawEvent(t *testing.T, bundle spcprovider.RawConvectiveOutlookBundle) event.Event {
|
func spcRawEvent(t *testing.T, bundle spcprovider.RawConvectiveOutlookBundle) event.Event {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
raw, err := json.Marshal(bundle)
|
raw, err := json.Marshal(bundle)
|
||||||
@@ -276,7 +424,7 @@ func spcRawEvent(t *testing.T, bundle spcprovider.RawConvectiveOutlookBundle) ev
|
|||||||
effectiveAt := time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)
|
effectiveAt := time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)
|
||||||
return event.Event{
|
return event.Event{
|
||||||
ID: "evt-spc-outlook-1",
|
ID: "evt-spc-outlook-1",
|
||||||
Kind: event.Kind("outlook"),
|
Kind: event.Kind(standards.KindOutlook),
|
||||||
Source: "spc-test",
|
Source: "spc-test",
|
||||||
EmittedAt: time.Date(2026, 6, 11, 20, 5, 0, 0, time.UTC),
|
EmittedAt: time.Date(2026, 6, 11, 20, 5, 0, 0, time.UTC),
|
||||||
EffectiveAt: &effectiveAt,
|
EffectiveAt: &effectiveAt,
|
||||||
@@ -322,23 +470,13 @@ func geoJSONFixtureForProduct(t *testing.T, key string) []byte {
|
|||||||
case strings.HasPrefix(key, "day2_"):
|
case strings.HasPrefix(key, "day2_"):
|
||||||
return readSPCTestFixture(t, "day2_torn.geojson")
|
return readSPCTestFixture(t, "day2_torn.geojson")
|
||||||
case strings.HasPrefix(key, "day3_"):
|
case strings.HasPrefix(key, "day3_"):
|
||||||
return readSPCTestFixture(t, "day3_wind.geojson")
|
return readSPCTestFixture(t, "day3_cat.geojson")
|
||||||
default:
|
default:
|
||||||
t.Fatalf("unknown product key %q", key)
|
t.Fatalf("unknown product key %q", key)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func readSPCTestFixture(t *testing.T, name string) []byte {
|
|
||||||
t.Helper()
|
|
||||||
path := filepath.Join("..", "..", "providers", "spc", "testdata", name)
|
|
||||||
raw, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read fixture %s: %v", path, err)
|
|
||||||
}
|
|
||||||
return raw
|
|
||||||
}
|
|
||||||
|
|
||||||
func findOutlook(outlooks []model.WeatherOutlook, day int, outlookType string) *model.WeatherOutlook {
|
func findOutlook(outlooks []model.WeatherOutlook, day int, outlookType string) *model.WeatherOutlook {
|
||||||
for i := range outlooks {
|
for i := range outlooks {
|
||||||
if outlooks[i].Day == day && outlooks[i].OutlookType == outlookType {
|
if outlooks[i].Day == day && outlooks[i].OutlookType == outlookType {
|
||||||
@@ -348,6 +486,37 @@ func findOutlook(outlooks []model.WeatherOutlook, day int, outlookType string) *
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func countOutlooksByDay(outlooks []model.WeatherOutlook, day int) int {
|
||||||
|
count := 0
|
||||||
|
for _, outlook := range outlooks {
|
||||||
|
if outlook.Day == day {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return count
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertAllOutlooksContainLocation(t *testing.T, outlooks []model.WeatherOutlook) {
|
||||||
|
t.Helper()
|
||||||
|
for i, outlook := range outlooks {
|
||||||
|
if !outlook.ContainsLocation {
|
||||||
|
t.Fatalf("outlook[%d].ContainsLocation = false, want true", i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertDiscussionDays(t *testing.T, discussions []model.WeatherOutlookDiscussion, want ...int) {
|
||||||
|
t.Helper()
|
||||||
|
if len(discussions) != len(want) {
|
||||||
|
t.Fatalf("Discussions length = %d, want %d", len(discussions), len(want))
|
||||||
|
}
|
||||||
|
for i, day := range want {
|
||||||
|
if discussions[i].Day != day {
|
||||||
|
t.Fatalf("Discussions[%d].Day = %d, want %d", i, discussions[i].Day, day)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func assertTime(t *testing.T, name string, got time.Time, year int, month time.Month, day int, hour int, minute int, second int) {
|
func assertTime(t *testing.T, name string, got time.Time, year int, month time.Month, day int, hour int, minute int, second int) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
want := time.Date(year, month, day, hour, minute, second, 0, time.UTC)
|
want := time.Date(year, month, day, hour, minute, second, 0, time.UTC)
|
||||||
|
|||||||
17
internal/normalizers/spc/fixture_test.go
Normal file
17
internal/normalizers/spc/fixture_test.go
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
package spc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func readSPCTestFixture(t *testing.T, name string) []byte {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join("..", "..", "providers", "spc", "testdata", name)
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read fixture %s: %v", path, err)
|
||||||
|
}
|
||||||
|
return raw
|
||||||
|
}
|
||||||
17
internal/providers/spc/fixture_test.go
Normal file
17
internal/providers/spc/fixture_test.go
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
package spc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func readTestFile(t *testing.T, name string) []byte {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join("testdata", name)
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read %s: %v", path, err)
|
||||||
|
}
|
||||||
|
return raw
|
||||||
|
}
|
||||||
@@ -33,6 +33,11 @@ type GeoJSONProperties struct {
|
|||||||
DN *int `json:"DN"`
|
DN *int `json:"DN"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type geometryMetadata struct {
|
||||||
|
Type string `json:"type"`
|
||||||
|
Geometries []json.RawMessage `json:"geometries"`
|
||||||
|
}
|
||||||
|
|
||||||
// DecodeGeoJSON decodes an SPC GeoJSON outlook product and compacts feature
|
// DecodeGeoJSON decodes an SPC GeoJSON outlook product and compacts feature
|
||||||
// geometry JSON for stable downstream storage.
|
// geometry JSON for stable downstream storage.
|
||||||
func DecodeGeoJSON(raw []byte) (GeoJSONFeatureCollection, error) {
|
func DecodeGeoJSON(raw []byte) (GeoJSONFeatureCollection, error) {
|
||||||
@@ -50,6 +55,16 @@ func DecodeGeoJSON(raw []byte) (GeoJSONFeatureCollection, error) {
|
|||||||
return collection, nil
|
return collection, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// IsEmptyGeometryCollection reports whether raw is SPC's no-polygon placeholder
|
||||||
|
// geometry shape: a GeometryCollection with no child geometries.
|
||||||
|
func IsEmptyGeometryCollection(raw json.RawMessage) bool {
|
||||||
|
var meta geometryMetadata
|
||||||
|
if err := json.Unmarshal(raw, &meta); err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return meta.Type == "GeometryCollection" && len(meta.Geometries) == 0
|
||||||
|
}
|
||||||
|
|
||||||
func (p *GeoJSONProperties) UnmarshalJSON(raw []byte) error {
|
func (p *GeoJSONProperties) UnmarshalJSON(raw []byte) error {
|
||||||
type alias GeoJSONProperties
|
type alias GeoJSONProperties
|
||||||
var aux struct {
|
var aux struct {
|
||||||
|
|||||||
@@ -1,8 +1,6 @@
|
|||||||
package spc
|
package spc
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -68,6 +66,43 @@ func TestDecodeGeoJSONParsesSeverityRankString(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestIsEmptyGeometryCollection(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
raw string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "empty geometry collection",
|
||||||
|
raw: `{"type":"GeometryCollection","geometries":[]}`,
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "non-empty geometry collection",
|
||||||
|
raw: `{"type":"GeometryCollection","geometries":[{"type":"Polygon","coordinates":[]} ]}`,
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "polygon",
|
||||||
|
raw: `{"type":"Polygon","coordinates":[]}`,
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid json",
|
||||||
|
raw: `{`,
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
if got := IsEmptyGeometryCollection([]byte(tt.raw)); got != tt.want {
|
||||||
|
t.Fatalf("IsEmptyGeometryCollection() = %v, want %v", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestParseISOTimestampTrimsAndReturnsUTC(t *testing.T) {
|
func TestParseISOTimestampTrimsAndReturnsUTC(t *testing.T) {
|
||||||
got, err := ParseISOTimestamp(" 2026-06-11T12:34:56Z ")
|
got, err := ParseISOTimestamp(" 2026-06-11T12:34:56Z ")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -78,13 +113,3 @@ func TestParseISOTimestampTrimsAndReturnsUTC(t *testing.T) {
|
|||||||
t.Fatalf("ParseISOTimestamp() = %s, want %s", got, want)
|
t.Fatalf("ParseISOTimestamp() = %s, want %s", got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func readTestFile(t *testing.T, name string) []byte {
|
|
||||||
t.Helper()
|
|
||||||
path := filepath.Join("testdata", name)
|
|
||||||
raw, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read %s: %v", path, err)
|
|
||||||
}
|
|
||||||
return raw
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -34,9 +34,6 @@ var geoJSONProducts = []GeoJSONProduct{
|
|||||||
{Key: "day2_hail", Day: 2, OutlookType: OutlookTypeHail, URL: "https://www.spc.noaa.gov/products/outlook/day2otlk_hail.nolyr.geojson"},
|
{Key: "day2_hail", Day: 2, OutlookType: OutlookTypeHail, URL: "https://www.spc.noaa.gov/products/outlook/day2otlk_hail.nolyr.geojson"},
|
||||||
{Key: "day2_wind", Day: 2, OutlookType: OutlookTypeWind, URL: "https://www.spc.noaa.gov/products/outlook/day2otlk_wind.nolyr.geojson"},
|
{Key: "day2_wind", Day: 2, OutlookType: OutlookTypeWind, URL: "https://www.spc.noaa.gov/products/outlook/day2otlk_wind.nolyr.geojson"},
|
||||||
{Key: "day3_categorical", Day: 3, OutlookType: OutlookTypeCategorical, URL: "https://www.spc.noaa.gov/products/outlook/day3otlk_cat.nolyr.geojson"},
|
{Key: "day3_categorical", Day: 3, OutlookType: OutlookTypeCategorical, URL: "https://www.spc.noaa.gov/products/outlook/day3otlk_cat.nolyr.geojson"},
|
||||||
{Key: "day3_tornado", Day: 3, OutlookType: OutlookTypeTornado, URL: "https://www.spc.noaa.gov/products/outlook/day3otlk_torn.nolyr.geojson"},
|
|
||||||
{Key: "day3_hail", Day: 3, OutlookType: OutlookTypeHail, URL: "https://www.spc.noaa.gov/products/outlook/day3otlk_hail.nolyr.geojson"},
|
|
||||||
{Key: "day3_wind", Day: 3, OutlookType: OutlookTypeWind, URL: "https://www.spc.noaa.gov/products/outlook/day3otlk_wind.nolyr.geojson"},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var discussionProducts = []DiscussionProduct{
|
var discussionProducts = []DiscussionProduct{
|
||||||
|
|||||||
@@ -4,8 +4,8 @@ import "testing"
|
|||||||
|
|
||||||
func TestGeoJSONProductsStableOrder(t *testing.T) {
|
func TestGeoJSONProductsStableOrder(t *testing.T) {
|
||||||
got := GeoJSONProducts()
|
got := GeoJSONProducts()
|
||||||
if len(got) != 12 {
|
if len(got) != 9 {
|
||||||
t.Fatalf("GeoJSONProducts() length = %d, want 12", len(got))
|
t.Fatalf("GeoJSONProducts() length = %d, want 9", len(got))
|
||||||
}
|
}
|
||||||
|
|
||||||
wantKeys := []string{
|
wantKeys := []string{
|
||||||
@@ -18,9 +18,6 @@ func TestGeoJSONProductsStableOrder(t *testing.T) {
|
|||||||
"day2_hail",
|
"day2_hail",
|
||||||
"day2_wind",
|
"day2_wind",
|
||||||
"day3_categorical",
|
"day3_categorical",
|
||||||
"day3_tornado",
|
|
||||||
"day3_hail",
|
|
||||||
"day3_wind",
|
|
||||||
}
|
}
|
||||||
for i, want := range wantKeys {
|
for i, want := range wantKeys {
|
||||||
if got[i].Key != want {
|
if got[i].Key != want {
|
||||||
|
|||||||
@@ -8,9 +8,9 @@
|
|||||||
"EXPIRE_ISO": "2026-06-14T12:00:00Z",
|
"EXPIRE_ISO": "2026-06-14T12:00:00Z",
|
||||||
"ISSUE_ISO": "2026-06-11T19:45:00Z",
|
"ISSUE_ISO": "2026-06-11T19:45:00Z",
|
||||||
"FORECASTER": "LEE",
|
"FORECASTER": "LEE",
|
||||||
"LABEL": "15",
|
"LABEL": "MRGL",
|
||||||
"LABEL2": "15% Wind",
|
"LABEL2": "Marginal Risk",
|
||||||
"DN": 15
|
"DN": 2
|
||||||
},
|
},
|
||||||
"geometry": {
|
"geometry": {
|
||||||
"type": "MultiPolygon",
|
"type": "MultiPolygon",
|
||||||
@@ -11,7 +11,7 @@
|
|||||||
// - weather.forecast_discussion.v1 -> model.WeatherForecastDiscussion
|
// - weather.forecast_discussion.v1 -> model.WeatherForecastDiscussion
|
||||||
// - weather.weather_story.v1 -> model.WeatherStoryRun
|
// - weather.weather_story.v1 -> model.WeatherStoryRun
|
||||||
// - weather.alert.v1 -> model.WeatherAlertRun
|
// - weather.alert.v1 -> model.WeatherAlertRun
|
||||||
// - weather.outlook.v1 -> model.WeatherOutlookRun
|
// - weather.outlook.v2 -> model.WeatherOutlookRun
|
||||||
//
|
//
|
||||||
// Parent/child relationships:
|
// Parent/child relationships:
|
||||||
// - observations.event_id -> observation_present_weather.event_id
|
// - observations.event_id -> observation_present_weather.event_id
|
||||||
@@ -21,6 +21,7 @@
|
|||||||
// - alert_runs.event_id -> alerts.run_event_id
|
// - alert_runs.event_id -> alerts.run_event_id
|
||||||
// - alerts.(run_event_id, alert_index) -> alert_references.(run_event_id, alert_index)
|
// - alerts.(run_event_id, alert_index) -> alert_references.(run_event_id, alert_index)
|
||||||
// - outlook_runs.event_id -> outlooks.run_event_id
|
// - outlook_runs.event_id -> outlooks.run_event_id
|
||||||
|
// - outlook_runs.event_id -> outlook_discussions.run_event_id
|
||||||
//
|
//
|
||||||
// Dedupe and retention behavior:
|
// Dedupe and retention behavior:
|
||||||
// - Parent primary keys (event_id): observations, forecasts, alert_runs, outlook_runs.
|
// - Parent primary keys (event_id): observations, forecasts, alert_runs, outlook_runs.
|
||||||
@@ -39,6 +40,7 @@
|
|||||||
// - alert_references.as_of
|
// - alert_references.as_of
|
||||||
// - outlook_runs.as_of
|
// - outlook_runs.as_of
|
||||||
// - outlooks.as_of
|
// - outlooks.as_of
|
||||||
|
// - outlook_discussions.as_of
|
||||||
//
|
//
|
||||||
// Envelope field mapping (shared parent columns)
|
// Envelope field mapping (shared parent columns)
|
||||||
//
|
//
|
||||||
@@ -227,6 +229,7 @@
|
|||||||
// - as_of TIMESTAMPTZ -> payload.asOf
|
// - as_of TIMESTAMPTZ -> payload.asOf
|
||||||
// - issued_at TIMESTAMPTZ NULL -> payload.issuedAt
|
// - issued_at TIMESTAMPTZ NULL -> payload.issuedAt
|
||||||
// - outlook_count INTEGER -> len(payload.outlooks)
|
// - outlook_count INTEGER -> len(payload.outlooks)
|
||||||
|
// - discussion_count INTEGER -> len(payload.discussions)
|
||||||
//
|
//
|
||||||
// 11. outlooks (PK: run_event_id, outlook_index)
|
// 11. outlooks (PK: run_event_id, outlook_index)
|
||||||
//
|
//
|
||||||
@@ -246,14 +249,22 @@
|
|||||||
// - issued_at TIMESTAMPTZ -> payload.outlooks[i].issuedAt
|
// - issued_at TIMESTAMPTZ -> payload.outlooks[i].issuedAt
|
||||||
// - expires_at TIMESTAMPTZ -> payload.outlooks[i].expiresAt
|
// - expires_at TIMESTAMPTZ -> payload.outlooks[i].expiresAt
|
||||||
// - forecaster TEXT NULL -> payload.outlooks[i].forecaster
|
// - forecaster TEXT NULL -> payload.outlooks[i].forecaster
|
||||||
// - headline TEXT NULL -> payload.outlooks[i].headline
|
|
||||||
// - summary TEXT NULL -> payload.outlooks[i].summary
|
|
||||||
// - discussion TEXT NULL -> payload.outlooks[i].discussion
|
|
||||||
// - source_url TEXT NULL -> payload.outlooks[i].sourceUrl
|
// - source_url TEXT NULL -> payload.outlooks[i].sourceUrl
|
||||||
// - image_url TEXT NULL -> payload.outlooks[i].imageUrl
|
// - image_url TEXT NULL -> payload.outlooks[i].imageUrl
|
||||||
// - contains_location BOOLEAN -> payload.outlooks[i].containsLocation
|
// - contains_location BOOLEAN -> payload.outlooks[i].containsLocation
|
||||||
// - geometry_json TEXT -> compact JSON payload.outlooks[i].geometry
|
// - geometry_json TEXT -> compact JSON payload.outlooks[i].geometry
|
||||||
//
|
//
|
||||||
|
// 12. outlook_discussions (PK: run_event_id, discussion_index)
|
||||||
|
//
|
||||||
|
// - run_event_id TEXT -> outlook_runs.event_id / payload.discussions[i]
|
||||||
|
// - discussion_index INTEGER -> i (array position in payload.discussions)
|
||||||
|
// - as_of TIMESTAMPTZ -> payload.asOf (copied from parent)
|
||||||
|
// - day INTEGER -> payload.discussions[i].day
|
||||||
|
// - headline TEXT NULL -> payload.discussions[i].headline
|
||||||
|
// - summary TEXT NULL -> payload.discussions[i].summary
|
||||||
|
// - discussion TEXT NULL -> payload.discussions[i].discussion
|
||||||
|
// - updated_at TIMESTAMPTZ NULL -> payload.discussions[i].updatedAt
|
||||||
|
//
|
||||||
// Reconstructing canonical JSON payloads
|
// Reconstructing canonical JSON payloads
|
||||||
//
|
//
|
||||||
// - WeatherObservation:
|
// - WeatherObservation:
|
||||||
@@ -274,6 +285,7 @@
|
|||||||
// ordered by reference_index to rebuild references per alert.
|
// ordered by reference_index to rebuild references per alert.
|
||||||
//
|
//
|
||||||
// - WeatherOutlookRun:
|
// - WeatherOutlookRun:
|
||||||
// read one row from outlook_runs, then join outlooks by run_event_id ordered
|
// read one row from outlook_runs, join outlooks by run_event_id ordered by
|
||||||
// by outlook_index to rebuild outlooks.
|
// outlook_index to rebuild outlooks, then join outlook_discussions by
|
||||||
|
// run_event_id ordered by discussion_index to rebuild discussions.
|
||||||
package postgres
|
package postgres
|
||||||
|
|||||||
33
internal/sinks/postgres/docs_test.go
Normal file
33
internal/sinks/postgres/docs_test.go
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
package postgres
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDocumentedOutlookDiscussionStorage(t *testing.T) {
|
||||||
|
for _, path := range []string{
|
||||||
|
"../../../docs/integrations/postgres.md",
|
||||||
|
"../../../docs/internal/postgres-sink.md",
|
||||||
|
} {
|
||||||
|
t.Run(path, func(t *testing.T) {
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile(%s) error = %v", path, err)
|
||||||
|
}
|
||||||
|
doc := string(raw)
|
||||||
|
|
||||||
|
for _, want := range []string{
|
||||||
|
tableOutlookDiscussions,
|
||||||
|
"discussion_count",
|
||||||
|
"discussion_index",
|
||||||
|
"weather.outlook.v2",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(doc, want) {
|
||||||
|
t.Fatalf("%s missing %q", path, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -27,7 +27,7 @@ func mapPostgresEvent(_ context.Context, e fkevent.Event) ([]fksinks.PostgresWri
|
|||||||
return mapWeatherStoryEvent(e)
|
return mapWeatherStoryEvent(e)
|
||||||
case standards.SchemaWeatherAlertV1:
|
case standards.SchemaWeatherAlertV1:
|
||||||
return mapAlertEvent(e)
|
return mapAlertEvent(e)
|
||||||
case standards.SchemaWeatherOutlookV1:
|
case standards.SchemaWeatherOutlookV2:
|
||||||
return mapOutlookEvent(e)
|
return mapOutlookEvent(e)
|
||||||
default:
|
default:
|
||||||
return nil, nil
|
return nil, nil
|
||||||
@@ -48,13 +48,7 @@ func mapObservationEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
|
|||||||
|
|
||||||
writes = append(writes, fksinks.PostgresWrite{
|
writes = append(writes, fksinks.PostgresWrite{
|
||||||
Table: tableObservations,
|
Table: tableObservations,
|
||||||
Values: map[string]any{
|
Values: parentEventValues(e, map[string]any{
|
||||||
"event_id": e.ID,
|
|
||||||
"event_kind": string(e.Kind),
|
|
||||||
"event_source": e.Source,
|
|
||||||
"event_schema": e.Schema,
|
|
||||||
"event_emitted_at": e.EmittedAt.UTC(),
|
|
||||||
"event_effective_at": nullableTime(e.EffectiveAt),
|
|
||||||
"station_id": nullableString(obs.StationID),
|
"station_id": nullableString(obs.StationID),
|
||||||
"station_name": nullableString(obs.StationName),
|
"station_name": nullableString(obs.StationName),
|
||||||
"observed_at": observedAt,
|
"observed_at": observedAt,
|
||||||
@@ -70,7 +64,7 @@ func mapObservationEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
|
|||||||
"visibility_meters": nullableFloat64(obs.VisibilityMeters),
|
"visibility_meters": nullableFloat64(obs.VisibilityMeters),
|
||||||
"relative_humidity_percent": nullableFloat64(obs.RelativeHumidityPercent),
|
"relative_humidity_percent": nullableFloat64(obs.RelativeHumidityPercent),
|
||||||
"apparent_temperature_c": nullableFloat64(obs.ApparentTemperatureC),
|
"apparent_temperature_c": nullableFloat64(obs.ApparentTemperatureC),
|
||||||
},
|
}),
|
||||||
})
|
})
|
||||||
|
|
||||||
for i, pw := range obs.PresentWeather {
|
for i, pw := range obs.PresentWeather {
|
||||||
@@ -109,23 +103,17 @@ func mapForecastEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
|
|||||||
|
|
||||||
writes = append(writes, fksinks.PostgresWrite{
|
writes = append(writes, fksinks.PostgresWrite{
|
||||||
Table: tableForecasts,
|
Table: tableForecasts,
|
||||||
Values: map[string]any{
|
Values: parentEventValues(e, map[string]any{
|
||||||
"event_id": e.ID,
|
"location_id": nullableString(run.LocationID),
|
||||||
"event_kind": string(e.Kind),
|
"location_name": nullableString(run.LocationName),
|
||||||
"event_source": e.Source,
|
"issued_at": issuedAt,
|
||||||
"event_schema": e.Schema,
|
"updated_at": nullableTime(run.UpdatedAt),
|
||||||
"event_emitted_at": e.EmittedAt.UTC(),
|
"product": string(run.Product),
|
||||||
"event_effective_at": nullableTime(e.EffectiveAt),
|
"latitude": nullableFloat64(run.Latitude),
|
||||||
"location_id": nullableString(run.LocationID),
|
"longitude": nullableFloat64(run.Longitude),
|
||||||
"location_name": nullableString(run.LocationName),
|
"elevation_meters": nullableFloat64(run.ElevationMeters),
|
||||||
"issued_at": issuedAt,
|
"period_count": len(run.Periods),
|
||||||
"updated_at": nullableTime(run.UpdatedAt),
|
}),
|
||||||
"product": string(run.Product),
|
|
||||||
"latitude": nullableFloat64(run.Latitude),
|
|
||||||
"longitude": nullableFloat64(run.Longitude),
|
|
||||||
"elevation_meters": nullableFloat64(run.ElevationMeters),
|
|
||||||
"period_count": len(run.Periods),
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
|
|
||||||
for i, p := range run.Periods {
|
for i, p := range run.Periods {
|
||||||
@@ -186,13 +174,7 @@ func mapForecastDiscussionEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error
|
|||||||
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.KeyMessages))
|
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.KeyMessages))
|
||||||
writes = append(writes, fksinks.PostgresWrite{
|
writes = append(writes, fksinks.PostgresWrite{
|
||||||
Table: tableForecastDiscussions,
|
Table: tableForecastDiscussions,
|
||||||
Values: map[string]any{
|
Values: parentEventValues(e, map[string]any{
|
||||||
"event_id": e.ID,
|
|
||||||
"event_kind": string(e.Kind),
|
|
||||||
"event_source": e.Source,
|
|
||||||
"event_schema": e.Schema,
|
|
||||||
"event_emitted_at": e.EmittedAt.UTC(),
|
|
||||||
"event_effective_at": nullableTime(e.EffectiveAt),
|
|
||||||
"office_id": nullableString(run.OfficeID),
|
"office_id": nullableString(run.OfficeID),
|
||||||
"office_name": nullableString(run.OfficeName),
|
"office_name": nullableString(run.OfficeName),
|
||||||
"issued_at": issuedAt,
|
"issued_at": issuedAt,
|
||||||
@@ -205,7 +187,7 @@ func mapForecastDiscussionEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error
|
|||||||
"long_term_issued_at": longTermIssuedAt,
|
"long_term_issued_at": longTermIssuedAt,
|
||||||
"long_term_text": longTermText,
|
"long_term_text": longTermText,
|
||||||
"key_message_count": len(run.KeyMessages),
|
"key_message_count": len(run.KeyMessages),
|
||||||
},
|
}),
|
||||||
})
|
})
|
||||||
|
|
||||||
for i, msg := range run.KeyMessages {
|
for i, msg := range run.KeyMessages {
|
||||||
@@ -236,17 +218,11 @@ func mapWeatherStoryEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
|
|||||||
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.Stories))
|
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.Stories))
|
||||||
writes = append(writes, fksinks.PostgresWrite{
|
writes = append(writes, fksinks.PostgresWrite{
|
||||||
Table: tableWeatherStoryRuns,
|
Table: tableWeatherStoryRuns,
|
||||||
Values: map[string]any{
|
Values: parentEventValues(e, map[string]any{
|
||||||
"event_id": e.ID,
|
"office_id": nullableString(run.OfficeID),
|
||||||
"event_kind": string(e.Kind),
|
"as_of": asOf,
|
||||||
"event_source": e.Source,
|
"story_count": len(run.Stories),
|
||||||
"event_schema": e.Schema,
|
}),
|
||||||
"event_emitted_at": e.EmittedAt.UTC(),
|
|
||||||
"event_effective_at": nullableTime(e.EffectiveAt),
|
|
||||||
"office_id": nullableString(run.OfficeID),
|
|
||||||
"as_of": asOf,
|
|
||||||
"story_count": len(run.Stories),
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
|
|
||||||
for i, story := range run.Stories {
|
for i, story := range run.Stories {
|
||||||
@@ -290,20 +266,14 @@ func mapAlertEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
|
|||||||
|
|
||||||
writes = append(writes, fksinks.PostgresWrite{
|
writes = append(writes, fksinks.PostgresWrite{
|
||||||
Table: tableAlertRuns,
|
Table: tableAlertRuns,
|
||||||
Values: map[string]any{
|
Values: parentEventValues(e, map[string]any{
|
||||||
"event_id": e.ID,
|
"location_id": nullableString(run.LocationID),
|
||||||
"event_kind": string(e.Kind),
|
"location_name": nullableString(run.LocationName),
|
||||||
"event_source": e.Source,
|
"as_of": asOf,
|
||||||
"event_schema": e.Schema,
|
"latitude": nullableFloat64(run.Latitude),
|
||||||
"event_emitted_at": e.EmittedAt.UTC(),
|
"longitude": nullableFloat64(run.Longitude),
|
||||||
"event_effective_at": nullableTime(e.EffectiveAt),
|
"alert_count": len(run.Alerts),
|
||||||
"location_id": nullableString(run.LocationID),
|
}),
|
||||||
"location_name": nullableString(run.LocationName),
|
|
||||||
"as_of": asOf,
|
|
||||||
"latitude": nullableFloat64(run.Latitude),
|
|
||||||
"longitude": nullableFloat64(run.Longitude),
|
|
||||||
"alert_count": len(run.Alerts),
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
|
|
||||||
for i, a := range run.Alerts {
|
for i, a := range run.Alerts {
|
||||||
@@ -369,24 +339,23 @@ func mapOutlookEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
asOf := run.AsOf.UTC()
|
asOf := run.AsOf.UTC()
|
||||||
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.Outlooks))
|
if err := validateOutlookDiscussions(run.Discussions); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.Outlooks)+len(run.Discussions))
|
||||||
writes = append(writes, fksinks.PostgresWrite{
|
writes = append(writes, fksinks.PostgresWrite{
|
||||||
Table: tableOutlookRuns,
|
Table: tableOutlookRuns,
|
||||||
Values: map[string]any{
|
Values: parentEventValues(e, map[string]any{
|
||||||
"event_id": e.ID,
|
"location_id": nullableString(run.LocationID),
|
||||||
"event_kind": string(e.Kind),
|
"location_name": nullableString(run.LocationName),
|
||||||
"event_source": e.Source,
|
"latitude": nullableFloat64(run.Latitude),
|
||||||
"event_schema": e.Schema,
|
"longitude": nullableFloat64(run.Longitude),
|
||||||
"event_emitted_at": e.EmittedAt.UTC(),
|
"as_of": asOf,
|
||||||
"event_effective_at": nullableTime(e.EffectiveAt),
|
"issued_at": nullableTime(run.IssuedAt),
|
||||||
"location_id": nullableString(run.LocationID),
|
"outlook_count": len(run.Outlooks),
|
||||||
"location_name": nullableString(run.LocationName),
|
"discussion_count": len(run.Discussions),
|
||||||
"latitude": nullableFloat64(run.Latitude),
|
}),
|
||||||
"longitude": nullableFloat64(run.Longitude),
|
|
||||||
"as_of": asOf,
|
|
||||||
"issued_at": nullableTime(run.IssuedAt),
|
|
||||||
"outlook_count": len(run.Outlooks),
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
|
|
||||||
for i, outlook := range run.Outlooks {
|
for i, outlook := range run.Outlooks {
|
||||||
@@ -417,9 +386,6 @@ func mapOutlookEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
|
|||||||
"issued_at": outlook.IssuedAt.UTC(),
|
"issued_at": outlook.IssuedAt.UTC(),
|
||||||
"expires_at": outlook.ExpiresAt.UTC(),
|
"expires_at": outlook.ExpiresAt.UTC(),
|
||||||
"forecaster": nullableString(outlook.Forecaster),
|
"forecaster": nullableString(outlook.Forecaster),
|
||||||
"headline": nullableString(outlook.Headline),
|
|
||||||
"summary": nullableString(outlook.Summary),
|
|
||||||
"discussion": nullableString(outlook.Discussion),
|
|
||||||
"source_url": nullableString(outlook.SourceURL),
|
"source_url": nullableString(outlook.SourceURL),
|
||||||
"image_url": nullableString(outlook.ImageURL),
|
"image_url": nullableString(outlook.ImageURL),
|
||||||
"contains_location": outlook.ContainsLocation,
|
"contains_location": outlook.ContainsLocation,
|
||||||
@@ -428,6 +394,22 @@ func mapOutlookEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for i, discussion := range run.Discussions {
|
||||||
|
writes = append(writes, fksinks.PostgresWrite{
|
||||||
|
Table: tableOutlookDiscussions,
|
||||||
|
Values: map[string]any{
|
||||||
|
"run_event_id": e.ID,
|
||||||
|
"discussion_index": i,
|
||||||
|
"as_of": asOf,
|
||||||
|
"day": discussion.Day,
|
||||||
|
"headline": nullableString(discussion.Headline),
|
||||||
|
"summary": nullableString(discussion.Summary),
|
||||||
|
"discussion": nullableString(discussion.Discussion),
|
||||||
|
"updated_at": nullableTime(discussion.UpdatedAt),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
return writes, nil
|
return writes, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -459,6 +441,28 @@ func validateOutlook(outlook model.WeatherOutlook, index int) error {
|
|||||||
if len(outlook.Geometry) == 0 {
|
if len(outlook.Geometry) == 0 {
|
||||||
return fmt.Errorf("decode outlook payload: outlooks[%d].geometry is required", index)
|
return fmt.Errorf("decode outlook payload: outlooks[%d].geometry is required", index)
|
||||||
}
|
}
|
||||||
|
if !outlook.ContainsLocation {
|
||||||
|
return fmt.Errorf("decode outlook payload: outlooks[%d].containsLocation must be true", index)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateOutlookDiscussions(discussions []model.WeatherOutlookDiscussion) error {
|
||||||
|
seenDays := map[int]int{}
|
||||||
|
for i, discussion := range discussions {
|
||||||
|
if discussion.Day < 1 || discussion.Day > 3 {
|
||||||
|
return fmt.Errorf("decode outlook payload: discussions[%d].day must be 1, 2, or 3", i)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(discussion.Headline) == "" &&
|
||||||
|
strings.TrimSpace(discussion.Summary) == "" &&
|
||||||
|
strings.TrimSpace(discussion.Discussion) == "" {
|
||||||
|
return fmt.Errorf("decode outlook payload: discussions[%d] headline, summary, or discussion is required", i)
|
||||||
|
}
|
||||||
|
if first, ok := seenDays[discussion.Day]; ok {
|
||||||
|
return fmt.Errorf("decode outlook payload: discussions[%d].day duplicates discussions[%d].day %d", i, first, discussion.Day)
|
||||||
|
}
|
||||||
|
seenDays[discussion.Day] = i
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -488,6 +492,21 @@ func decodePayload[T any](payload any) (T, error) {
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func parentEventValues(e fkevent.Event, values map[string]any) map[string]any {
|
||||||
|
out := map[string]any{
|
||||||
|
"event_id": e.ID,
|
||||||
|
"event_kind": string(e.Kind),
|
||||||
|
"event_source": e.Source,
|
||||||
|
"event_schema": e.Schema,
|
||||||
|
"event_emitted_at": e.EmittedAt.UTC(),
|
||||||
|
"event_effective_at": nullableTime(e.EffectiveAt),
|
||||||
|
}
|
||||||
|
for k, v := range values {
|
||||||
|
out[k] = v
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func nullableDiscussionSection(section *model.WeatherForecastDiscussionSection) (any, any, any) {
|
func nullableDiscussionSection(section *model.WeatherForecastDiscussionSection) (any, any, any) {
|
||||||
if section == nil {
|
if section == nil {
|
||||||
return nil, nil, nil
|
return nil, nil, nil
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ func TestMapPostgresEventObservationStructPayload(t *testing.T) {
|
|||||||
PresentWeather: []model.PresentWeather{{Raw: map[string]any{"a": 1, "b": "x"}}},
|
PresentWeather: []model.PresentWeather{{Raw: map[string]any{"a": 1, "b": "x"}}},
|
||||||
}
|
}
|
||||||
|
|
||||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherObservationV1, "observation", obs))
|
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherObservationV1, standards.KindObservation, obs))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("mapPostgresEvent() error = %v", err)
|
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -74,7 +74,7 @@ func TestMapPostgresEventForecastStructPayload(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, "forecast", run))
|
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, standards.KindForecast, run))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("mapPostgresEvent() error = %v", err)
|
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -122,7 +122,7 @@ func TestMapPostgresEventAlertStructPayload(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherAlertV1, "alert", run))
|
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherAlertV1, standards.KindAlert, run))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("mapPostgresEvent() error = %v", err)
|
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -163,7 +163,7 @@ func TestMapPostgresEventForecastDiscussionStructPayload(t *testing.T) {
|
|||||||
LongTerm: &model.WeatherForecastDiscussionSection{Text: "Long term text"},
|
LongTerm: &model.WeatherForecastDiscussionSection{Text: "Long term text"},
|
||||||
}
|
}
|
||||||
|
|
||||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, "forecast_discussion", run))
|
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, standards.KindForecastDiscussion, run))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("mapPostgresEvent() error = %v", err)
|
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -213,7 +213,7 @@ func TestMapPostgresEventWeatherStoryStructPayload(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, "weather_story", run))
|
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, standards.KindWeatherStory, run))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("mapPostgresEvent() error = %v", err)
|
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -243,6 +243,7 @@ func TestMapPostgresEventOutlookStructPayload(t *testing.T) {
|
|||||||
lat := 38.6239
|
lat := 38.6239
|
||||||
lon := -90.3571
|
lon := -90.3571
|
||||||
issuedAt := time.Date(2026, 6, 11, 19, 45, 0, 0, time.FixedZone("UTC-5", -5*60*60))
|
issuedAt := time.Date(2026, 6, 11, 19, 45, 0, 0, time.FixedZone("UTC-5", -5*60*60))
|
||||||
|
updatedAt := time.Date(2026, 6, 11, 21, 15, 0, 0, time.FixedZone("UTC-5", -5*60*60))
|
||||||
severity := 3
|
severity := 3
|
||||||
run := model.WeatherOutlookRun{
|
run := model.WeatherOutlookRun{
|
||||||
LocationID: "stl",
|
LocationID: "stl",
|
||||||
@@ -266,9 +267,6 @@ func TestMapPostgresEventOutlookStructPayload(t *testing.T) {
|
|||||||
IssuedAt: issuedAt,
|
IssuedAt: issuedAt,
|
||||||
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||||
Forecaster: "SMITH",
|
Forecaster: "SMITH",
|
||||||
Headline: "Day 1 Convective Outlook",
|
|
||||||
Summary: "Severe thunderstorms are possible.",
|
|
||||||
Discussion: "Full discussion text.",
|
|
||||||
SourceURL: "https://example.invalid/day1.geojson",
|
SourceURL: "https://example.invalid/day1.geojson",
|
||||||
ContainsLocation: true,
|
ContainsLocation: true,
|
||||||
Geometry: json.RawMessage(`{ "type" : "Polygon", "coordinates" : [ [ [ -91.0, 38.0 ], [ -90.0, 38.0 ], [ -90.0, 39.0 ], [ -91.0, 39.0 ], [ -91.0, 38.0 ] ] ] }`),
|
Geometry: json.RawMessage(`{ "type" : "Polygon", "coordinates" : [ [ [ -91.0, 38.0 ], [ -90.0, 38.0 ], [ -90.0, 39.0 ], [ -91.0, 39.0 ], [ -91.0, 38.0 ] ] ] }`),
|
||||||
@@ -284,18 +282,27 @@ func TestMapPostgresEventOutlookStructPayload(t *testing.T) {
|
|||||||
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||||
IssuedAt: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
IssuedAt: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||||
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||||
ContainsLocation: false,
|
ContainsLocation: true,
|
||||||
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-100,35],[-98,35],[-98,37],[-100,37],[-100,35]]]}`),
|
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-100,35],[-98,35],[-98,37],[-100,37],[-100,35]]]}`),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
Discussions: []model.WeatherOutlookDiscussion{
|
||||||
|
{
|
||||||
|
Day: 1,
|
||||||
|
Headline: "Day 1 Convective Outlook",
|
||||||
|
Summary: "Severe thunderstorms are possible.",
|
||||||
|
Discussion: "Full discussion text.",
|
||||||
|
UpdatedAt: &updatedAt,
|
||||||
|
},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, "outlook", run))
|
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("mapPostgresEvent() error = %v", err)
|
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||||
}
|
}
|
||||||
if len(writes) != 3 {
|
if len(writes) != 4 {
|
||||||
t.Fatalf("mapPostgresEvent() writes len = %d, want 3", len(writes))
|
t.Fatalf("mapPostgresEvent() writes len = %d, want 4", len(writes))
|
||||||
}
|
}
|
||||||
if writes[0].Table != tableOutlookRuns {
|
if writes[0].Table != tableOutlookRuns {
|
||||||
t.Fatalf("writes[0].Table = %q, want %q", writes[0].Table, tableOutlookRuns)
|
t.Fatalf("writes[0].Table = %q, want %q", writes[0].Table, tableOutlookRuns)
|
||||||
@@ -303,6 +310,9 @@ func TestMapPostgresEventOutlookStructPayload(t *testing.T) {
|
|||||||
if got := writes[0].Values["outlook_count"]; got != 2 {
|
if got := writes[0].Values["outlook_count"]; got != 2 {
|
||||||
t.Fatalf("outlook_runs outlook_count = %#v, want 2", got)
|
t.Fatalf("outlook_runs outlook_count = %#v, want 2", got)
|
||||||
}
|
}
|
||||||
|
if got := writes[0].Values["discussion_count"]; got != 1 {
|
||||||
|
t.Fatalf("outlook_runs discussion_count = %#v, want 1", got)
|
||||||
|
}
|
||||||
if got := writes[0].Values["issued_at"]; got != issuedAt.UTC() {
|
if got := writes[0].Values["issued_at"]; got != issuedAt.UTC() {
|
||||||
t.Fatalf("outlook_runs issued_at = %#v, want UTC %s", got, issuedAt.UTC())
|
t.Fatalf("outlook_runs issued_at = %#v, want UTC %s", got, issuedAt.UTC())
|
||||||
}
|
}
|
||||||
@@ -324,15 +334,64 @@ func TestMapPostgresEventOutlookStructPayload(t *testing.T) {
|
|||||||
if got := writes[1].Values["geometry_json"]; got != `{"type":"Polygon","coordinates":[[[-91.0,38.0],[-90.0,38.0],[-90.0,39.0],[-91.0,39.0],[-91.0,38.0]]]}` {
|
if got := writes[1].Values["geometry_json"]; got != `{"type":"Polygon","coordinates":[[[-91.0,38.0],[-90.0,38.0],[-90.0,39.0],[-91.0,39.0],[-91.0,38.0]]]}` {
|
||||||
t.Fatalf("first geometry_json = %#v", got)
|
t.Fatalf("first geometry_json = %#v", got)
|
||||||
}
|
}
|
||||||
if got := writes[2].Values["contains_location"]; got != false {
|
if got := writes[2].Values["contains_location"]; got != true {
|
||||||
t.Fatalf("second contains_location = %#v, want false", got)
|
t.Fatalf("second contains_location = %#v, want true", got)
|
||||||
|
}
|
||||||
|
if writes[3].Table != tableOutlookDiscussions {
|
||||||
|
t.Fatalf("writes[3].Table = %q, want %q", writes[3].Table, tableOutlookDiscussions)
|
||||||
|
}
|
||||||
|
if got := writes[3].Values["discussion_index"]; got != 0 {
|
||||||
|
t.Fatalf("discussion_index = %#v, want 0", got)
|
||||||
|
}
|
||||||
|
if got := writes[3].Values["as_of"]; got != run.AsOf.UTC() {
|
||||||
|
t.Fatalf("discussion as_of = %#v, want %s", got, run.AsOf.UTC())
|
||||||
|
}
|
||||||
|
if got := writes[3].Values["day"]; got != 1 {
|
||||||
|
t.Fatalf("discussion day = %#v, want 1", got)
|
||||||
|
}
|
||||||
|
if got := writes[3].Values["headline"]; got != "Day 1 Convective Outlook" {
|
||||||
|
t.Fatalf("discussion headline = %#v", got)
|
||||||
|
}
|
||||||
|
if got := writes[3].Values["summary"]; got != "Severe thunderstorms are possible." {
|
||||||
|
t.Fatalf("discussion summary = %#v", got)
|
||||||
|
}
|
||||||
|
if got := writes[3].Values["discussion"]; got != "Full discussion text." {
|
||||||
|
t.Fatalf("discussion text = %#v", got)
|
||||||
|
}
|
||||||
|
if got := writes[3].Values["updated_at"]; got != updatedAt.UTC() {
|
||||||
|
t.Fatalf("discussion updated_at = %#v, want UTC %s", got, updatedAt.UTC())
|
||||||
|
}
|
||||||
|
|
||||||
|
assertAllWritesIncludeAllColumns(t, writes)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMapPostgresEventOutlookEmptyLocalRun(t *testing.T) {
|
||||||
|
run := model.WeatherOutlookRun{
|
||||||
|
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||||
|
}
|
||||||
|
|
||||||
|
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(writes) != 1 {
|
||||||
|
t.Fatalf("mapPostgresEvent() writes len = %d, want 1", len(writes))
|
||||||
|
}
|
||||||
|
if writes[0].Table != tableOutlookRuns {
|
||||||
|
t.Fatalf("writes[0].Table = %q, want %q", writes[0].Table, tableOutlookRuns)
|
||||||
|
}
|
||||||
|
if got := writes[0].Values["outlook_count"]; got != 0 {
|
||||||
|
t.Fatalf("outlook_runs outlook_count = %#v, want 0", got)
|
||||||
|
}
|
||||||
|
if got := writes[0].Values["discussion_count"]; got != 0 {
|
||||||
|
t.Fatalf("outlook_runs discussion_count = %#v, want 0", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
assertAllWritesIncludeAllColumns(t, writes)
|
assertAllWritesIncludeAllColumns(t, writes)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMapPostgresEventOutlookRejectsMissingAsOf(t *testing.T) {
|
func TestMapPostgresEventOutlookRejectsMissingAsOf(t *testing.T) {
|
||||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, "outlook", model.WeatherOutlookRun{}))
|
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, model.WeatherOutlookRun{}))
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatalf("mapPostgresEvent() error = nil, want missing asOf error")
|
t.Fatalf("mapPostgresEvent() error = nil, want missing asOf error")
|
||||||
}
|
}
|
||||||
@@ -343,17 +402,18 @@ func TestMapPostgresEventOutlookRejectsMissingAsOf(t *testing.T) {
|
|||||||
|
|
||||||
func TestMapPostgresEventOutlookRejectsMissingIDAndProvider(t *testing.T) {
|
func TestMapPostgresEventOutlookRejectsMissingIDAndProvider(t *testing.T) {
|
||||||
base := model.WeatherOutlook{
|
base := model.WeatherOutlook{
|
||||||
ID: "outlook-1",
|
ID: "outlook-1",
|
||||||
Provider: "spc",
|
Provider: "spc",
|
||||||
Product: "convective",
|
Product: "convective",
|
||||||
Day: 1,
|
Day: 1,
|
||||||
OutlookType: "categorical",
|
OutlookType: "categorical",
|
||||||
Label: "SLGT",
|
Label: "SLGT",
|
||||||
ValidFrom: time.Date(2026, 6, 11, 13, 0, 0, 0, time.UTC),
|
ValidFrom: time.Date(2026, 6, 11, 13, 0, 0, 0, time.UTC),
|
||||||
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||||
IssuedAt: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
IssuedAt: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||||
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||||
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-91,38],[-90,38],[-90,39],[-91,39],[-91,38]]]}`),
|
ContainsLocation: true,
|
||||||
|
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-91,38],[-90,38],[-90,39],[-91,39],[-91,38]]]}`),
|
||||||
}
|
}
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
@@ -381,7 +441,7 @@ func TestMapPostgresEventOutlookRejectsMissingIDAndProvider(t *testing.T) {
|
|||||||
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||||
Outlooks: []model.WeatherOutlook{outlook},
|
Outlooks: []model.WeatherOutlook{outlook},
|
||||||
}
|
}
|
||||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, "outlook", run))
|
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatalf("mapPostgresEvent() error = nil, want %q", tt.wantErr)
|
t.Fatalf("mapPostgresEvent() error = nil, want %q", tt.wantErr)
|
||||||
}
|
}
|
||||||
@@ -396,16 +456,17 @@ func TestMapPostgresEventOutlookRejectsMissingRequiredTimes(t *testing.T) {
|
|||||||
run := model.WeatherOutlookRun{
|
run := model.WeatherOutlookRun{
|
||||||
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||||
Outlooks: []model.WeatherOutlook{{
|
Outlooks: []model.WeatherOutlook{{
|
||||||
ID: "outlook-1",
|
ID: "outlook-1",
|
||||||
Provider: "spc",
|
Provider: "spc",
|
||||||
Product: "convective",
|
Product: "convective",
|
||||||
Day: 1,
|
Day: 1,
|
||||||
OutlookType: "categorical",
|
OutlookType: "categorical",
|
||||||
Label: "SLGT",
|
Label: "SLGT",
|
||||||
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-91,38],[-90,38],[-90,39],[-91,39],[-91,38]]]}`),
|
ContainsLocation: true,
|
||||||
|
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-91,38],[-90,38],[-90,39],[-91,39],[-91,38]]]}`),
|
||||||
}},
|
}},
|
||||||
}
|
}
|
||||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, "outlook", run))
|
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatalf("mapPostgresEvent() error = nil, want missing time error")
|
t.Fatalf("mapPostgresEvent() error = nil, want missing time error")
|
||||||
}
|
}
|
||||||
@@ -418,19 +479,20 @@ func TestMapPostgresEventOutlookRejectsEmptyGeometry(t *testing.T) {
|
|||||||
run := model.WeatherOutlookRun{
|
run := model.WeatherOutlookRun{
|
||||||
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||||
Outlooks: []model.WeatherOutlook{{
|
Outlooks: []model.WeatherOutlook{{
|
||||||
ID: "outlook-1",
|
ID: "outlook-1",
|
||||||
Provider: "spc",
|
Provider: "spc",
|
||||||
Product: "convective",
|
Product: "convective",
|
||||||
Day: 1,
|
Day: 1,
|
||||||
OutlookType: "categorical",
|
OutlookType: "categorical",
|
||||||
Label: "SLGT",
|
Label: "SLGT",
|
||||||
ValidFrom: time.Date(2026, 6, 11, 13, 0, 0, 0, time.UTC),
|
ValidFrom: time.Date(2026, 6, 11, 13, 0, 0, 0, time.UTC),
|
||||||
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||||
IssuedAt: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
IssuedAt: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||||
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||||
|
ContainsLocation: true,
|
||||||
}},
|
}},
|
||||||
}
|
}
|
||||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, "outlook", run))
|
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatalf("mapPostgresEvent() error = nil, want geometry error")
|
t.Fatalf("mapPostgresEvent() error = nil, want geometry error")
|
||||||
}
|
}
|
||||||
@@ -439,8 +501,69 @@ func TestMapPostgresEventOutlookRejectsEmptyGeometry(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMapPostgresEventOutlookRejectsDuplicateDiscussionDay(t *testing.T) {
|
||||||
|
run := model.WeatherOutlookRun{
|
||||||
|
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||||
|
Discussions: []model.WeatherOutlookDiscussion{
|
||||||
|
{Day: 1, Discussion: "First day one discussion."},
|
||||||
|
{Day: 1, Discussion: "Duplicate day one discussion."},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("mapPostgresEvent() error = nil, want duplicate discussion day error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "discussions[1].day duplicates discussions[0].day 1") {
|
||||||
|
t.Fatalf("error = %q, want duplicate discussion day context", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMapPostgresEventOutlookRejectsInvalidDiscussionDay(t *testing.T) {
|
||||||
|
run := model.WeatherOutlookRun{
|
||||||
|
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||||
|
Discussions: []model.WeatherOutlookDiscussion{{Day: 4, Discussion: "Invalid day."}},
|
||||||
|
}
|
||||||
|
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("mapPostgresEvent() error = nil, want invalid discussion day error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "discussions[0].day must be 1, 2, or 3") {
|
||||||
|
t.Fatalf("error = %q, want invalid discussion day context", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMapPostgresEventOutlookRejectsEmptyDiscussionContent(t *testing.T) {
|
||||||
|
run := model.WeatherOutlookRun{
|
||||||
|
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||||
|
Discussions: []model.WeatherOutlookDiscussion{{Day: 1}},
|
||||||
|
}
|
||||||
|
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("mapPostgresEvent() error = nil, want empty discussion content error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "discussions[0] headline, summary, or discussion is required") {
|
||||||
|
t.Fatalf("error = %q, want empty discussion content context", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMapPostgresEventOutlookRejectsContainsLocationFalse(t *testing.T) {
|
||||||
|
run := model.WeatherOutlookRun{
|
||||||
|
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||||
|
Outlooks: []model.WeatherOutlook{validTestOutlook()},
|
||||||
|
}
|
||||||
|
run.Outlooks[0].ContainsLocation = false
|
||||||
|
|
||||||
|
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV2, standards.KindOutlook, run))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("mapPostgresEvent() error = nil, want containsLocation error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "outlooks[0].containsLocation must be true") {
|
||||||
|
t.Fatalf("error = %q, want containsLocation context", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestMapPostgresEventWeatherStoryRejectsMissingAsOf(t *testing.T) {
|
func TestMapPostgresEventWeatherStoryRejectsMissingAsOf(t *testing.T) {
|
||||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, "weather_story", model.WeatherStoryRun{}))
|
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, standards.KindWeatherStory, model.WeatherStoryRun{}))
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatalf("mapPostgresEvent() error = nil, want missing asOf error")
|
t.Fatalf("mapPostgresEvent() error = nil, want missing asOf error")
|
||||||
}
|
}
|
||||||
@@ -454,7 +577,7 @@ func TestMapPostgresEventWeatherStoryRejectsMissingStoryTimes(t *testing.T) {
|
|||||||
AsOf: time.Date(2026, 5, 30, 9, 0, 34, 0, time.UTC),
|
AsOf: time.Date(2026, 5, 30, 9, 0, 34, 0, time.UTC),
|
||||||
Stories: []model.WeatherStory{{Title: "missing times"}},
|
Stories: []model.WeatherStory{{Title: "missing times"}},
|
||||||
}
|
}
|
||||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, "weather_story", run))
|
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, standards.KindWeatherStory, run))
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatalf("mapPostgresEvent() error = nil, want missing story times error")
|
t.Fatalf("mapPostgresEvent() error = nil, want missing story times error")
|
||||||
}
|
}
|
||||||
@@ -484,7 +607,7 @@ func TestMapPostgresEventMapPayload(t *testing.T) {
|
|||||||
t.Fatalf("json.Unmarshal() error = %v", err)
|
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, "forecast", payload))
|
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, standards.KindForecast, payload))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("mapPostgresEvent() error = %v", err)
|
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -499,7 +622,18 @@ func TestMapPostgresEventMapPayload(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestMapPostgresEventUnknownSchemaNoOp(t *testing.T) {
|
func TestMapPostgresEventUnknownSchemaNoOp(t *testing.T) {
|
||||||
writes, err := mapPostgresEvent(context.Background(), testEvent("weather.unknown.v1", "observation", map[string]any{"x": 1}))
|
writes, err := mapPostgresEvent(context.Background(), testEvent("weather.unknown.v1", standards.KindObservation, map[string]any{"x": 1}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(writes) != 0 {
|
||||||
|
t.Fatalf("mapPostgresEvent() writes len = %d, want 0", len(writes))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMapPostgresEventLegacyOutlookSchemaNoOp(t *testing.T) {
|
||||||
|
run := model.WeatherOutlookRun{AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)}
|
||||||
|
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, standards.KindOutlook, run))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("mapPostgresEvent() error = %v", err)
|
t.Fatalf("mapPostgresEvent() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -509,7 +643,7 @@ func TestMapPostgresEventUnknownSchemaNoOp(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestMapPostgresEventMalformedPayload(t *testing.T) {
|
func TestMapPostgresEventMalformedPayload(t *testing.T) {
|
||||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, "forecast", "bad"))
|
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, standards.KindForecast, "bad"))
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatalf("mapPostgresEvent() expected error for malformed payload")
|
t.Fatalf("mapPostgresEvent() expected error for malformed payload")
|
||||||
}
|
}
|
||||||
@@ -519,7 +653,7 @@ func TestMapPostgresEventMalformedPayload(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestMapPostgresEventForecastDiscussionMalformedPayload(t *testing.T) {
|
func TestMapPostgresEventForecastDiscussionMalformedPayload(t *testing.T) {
|
||||||
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, "forecast_discussion", "bad"))
|
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, standards.KindForecastDiscussion, "bad"))
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatalf("mapPostgresEvent() expected error for malformed payload")
|
t.Fatalf("mapPostgresEvent() expected error for malformed payload")
|
||||||
}
|
}
|
||||||
@@ -528,6 +662,56 @@ func TestMapPostgresEventForecastDiscussionMalformedPayload(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParentEventValuesAddsEnvelopeAndPreservesProductValues(t *testing.T) {
|
||||||
|
emittedAt := time.Date(2026, 3, 16, 13, 31, 0, 0, time.FixedZone("CDT", -5*60*60))
|
||||||
|
effectiveAt := time.Date(2026, 3, 16, 13, 30, 0, 0, time.FixedZone("CDT", -5*60*60))
|
||||||
|
event := fkevent.Event{
|
||||||
|
ID: "evt-envelope",
|
||||||
|
Kind: fkevent.Kind(standards.KindForecast),
|
||||||
|
Source: "test-source",
|
||||||
|
Schema: standards.SchemaWeatherForecastV1,
|
||||||
|
EmittedAt: emittedAt,
|
||||||
|
EffectiveAt: &effectiveAt,
|
||||||
|
}
|
||||||
|
|
||||||
|
got := parentEventValues(event, map[string]any{"product_col": "product-value"})
|
||||||
|
|
||||||
|
assertParentEnvelopeValues(t, got, event)
|
||||||
|
if got["product_col"] != "product-value" {
|
||||||
|
t.Fatalf("product_col = %#v, want product-value", got["product_col"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParentEventValuesNullEffectiveAt(t *testing.T) {
|
||||||
|
base := fkevent.Event{
|
||||||
|
ID: "evt-envelope",
|
||||||
|
Kind: fkevent.Kind(standards.KindObservation),
|
||||||
|
Source: "test-source",
|
||||||
|
Schema: standards.SchemaWeatherObservationV1,
|
||||||
|
EmittedAt: time.Date(2026, 3, 16, 18, 31, 0, 0, time.UTC),
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range []struct {
|
||||||
|
name string
|
||||||
|
mut func(*fkevent.Event)
|
||||||
|
}{
|
||||||
|
{name: "nil", mut: func(*fkevent.Event) {}},
|
||||||
|
{name: "zero", mut: func(event *fkevent.Event) {
|
||||||
|
zero := time.Time{}
|
||||||
|
event.EffectiveAt = &zero
|
||||||
|
}},
|
||||||
|
} {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
event := base
|
||||||
|
tt.mut(&event)
|
||||||
|
got := parentEventValues(event, nil)
|
||||||
|
if got["event_effective_at"] != nil {
|
||||||
|
t.Fatalf("event_effective_at = %#v, want nil", got["event_effective_at"])
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func testEvent(schema string, kind fkevent.Kind, payload any) fkevent.Event {
|
func testEvent(schema string, kind fkevent.Kind, payload any) fkevent.Event {
|
||||||
effectiveAt := time.Date(2026, 3, 16, 18, 30, 0, 0, time.UTC)
|
effectiveAt := time.Date(2026, 3, 16, 18, 30, 0, 0, time.UTC)
|
||||||
return fkevent.Event{
|
return fkevent.Event{
|
||||||
@@ -541,6 +725,30 @@ func testEvent(schema string, kind fkevent.Kind, payload any) fkevent.Event {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func assertParentEnvelopeValues(t *testing.T, values map[string]any, event fkevent.Event) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
if got := values["event_id"]; got != event.ID {
|
||||||
|
t.Fatalf("event_id = %#v, want %q", got, event.ID)
|
||||||
|
}
|
||||||
|
if got := values["event_kind"]; got != string(event.Kind) {
|
||||||
|
t.Fatalf("event_kind = %#v, want %q", got, event.Kind)
|
||||||
|
}
|
||||||
|
if got := values["event_source"]; got != event.Source {
|
||||||
|
t.Fatalf("event_source = %#v, want %q", got, event.Source)
|
||||||
|
}
|
||||||
|
if got := values["event_schema"]; got != event.Schema {
|
||||||
|
t.Fatalf("event_schema = %#v, want %q", got, event.Schema)
|
||||||
|
}
|
||||||
|
if got := values["event_emitted_at"]; got != event.EmittedAt.UTC() {
|
||||||
|
t.Fatalf("event_emitted_at = %#v, want %s", got, event.EmittedAt.UTC())
|
||||||
|
}
|
||||||
|
wantEffective := nullableTime(event.EffectiveAt)
|
||||||
|
if got := values["event_effective_at"]; got != wantEffective {
|
||||||
|
t.Fatalf("event_effective_at = %#v, want %#v", got, wantEffective)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func firstWriteForTable(writes []fksinks.PostgresWrite, table string) (fksinks.PostgresWrite, bool) {
|
func firstWriteForTable(writes []fksinks.PostgresWrite, table string) (fksinks.PostgresWrite, bool) {
|
||||||
for _, w := range writes {
|
for _, w := range writes {
|
||||||
if w.Table == table {
|
if w.Table == table {
|
||||||
@@ -573,6 +781,23 @@ func tableColumnCounts() map[string]int {
|
|||||||
return m
|
return m
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validTestOutlook() model.WeatherOutlook {
|
||||||
|
return model.WeatherOutlook{
|
||||||
|
ID: "outlook-1",
|
||||||
|
Provider: "spc",
|
||||||
|
Product: "convective",
|
||||||
|
Day: 1,
|
||||||
|
OutlookType: "categorical",
|
||||||
|
Label: "SLGT",
|
||||||
|
ValidFrom: time.Date(2026, 6, 11, 13, 0, 0, 0, time.UTC),
|
||||||
|
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||||
|
IssuedAt: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||||
|
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||||
|
ContainsLocation: true,
|
||||||
|
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-91,38],[-90,38],[-90,39],[-91,39],[-91,38]]]}`),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func wmoCodePtr(v model.WMOCode) *model.WMOCode {
|
func wmoCodePtr(v model.WMOCode) *model.WMOCode {
|
||||||
out := v
|
out := v
|
||||||
return &out
|
return &out
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ const (
|
|||||||
tableAlertReferences = "alert_references"
|
tableAlertReferences = "alert_references"
|
||||||
tableOutlookRuns = "outlook_runs"
|
tableOutlookRuns = "outlook_runs"
|
||||||
tableOutlooks = "outlooks"
|
tableOutlooks = "outlooks"
|
||||||
|
tableOutlookDiscussions = "outlook_discussions"
|
||||||
)
|
)
|
||||||
|
|
||||||
// PostgresSchema returns weatherfeeder's Postgres schema definition.
|
// PostgresSchema returns weatherfeeder's Postgres schema definition.
|
||||||
@@ -26,13 +27,7 @@ func PostgresSchema() fksinks.PostgresSchema {
|
|||||||
Tables: []fksinks.PostgresTable{
|
Tables: []fksinks.PostgresTable{
|
||||||
{
|
{
|
||||||
Name: tableObservations,
|
Name: tableObservations,
|
||||||
Columns: []fksinks.PostgresColumn{
|
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
|
||||||
{Name: "event_id", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_kind", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_source", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_schema", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
|
|
||||||
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
|
|
||||||
{Name: "station_id", Type: "TEXT", Nullable: true},
|
{Name: "station_id", Type: "TEXT", Nullable: true},
|
||||||
{Name: "station_name", Type: "TEXT", Nullable: true},
|
{Name: "station_name", Type: "TEXT", Nullable: true},
|
||||||
{Name: "observed_at", Type: "TIMESTAMPTZ", Nullable: false},
|
{Name: "observed_at", Type: "TIMESTAMPTZ", Nullable: false},
|
||||||
@@ -48,7 +43,7 @@ func PostgresSchema() fksinks.PostgresSchema {
|
|||||||
{Name: "visibility_meters", Type: "DOUBLE PRECISION", Nullable: true},
|
{Name: "visibility_meters", Type: "DOUBLE PRECISION", Nullable: true},
|
||||||
{Name: "relative_humidity_percent", Type: "DOUBLE PRECISION", Nullable: true},
|
{Name: "relative_humidity_percent", Type: "DOUBLE PRECISION", Nullable: true},
|
||||||
{Name: "apparent_temperature_c", Type: "DOUBLE PRECISION", Nullable: true},
|
{Name: "apparent_temperature_c", Type: "DOUBLE PRECISION", Nullable: true},
|
||||||
},
|
}...),
|
||||||
PrimaryKey: []string{"event_id"},
|
PrimaryKey: []string{"event_id"},
|
||||||
PruneColumn: "observed_at",
|
PruneColumn: "observed_at",
|
||||||
Indexes: []fksinks.PostgresIndex{
|
Indexes: []fksinks.PostgresIndex{
|
||||||
@@ -73,13 +68,7 @@ func PostgresSchema() fksinks.PostgresSchema {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
Name: tableForecasts,
|
Name: tableForecasts,
|
||||||
Columns: []fksinks.PostgresColumn{
|
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
|
||||||
{Name: "event_id", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_kind", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_source", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_schema", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
|
|
||||||
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
|
|
||||||
{Name: "location_id", Type: "TEXT", Nullable: true},
|
{Name: "location_id", Type: "TEXT", Nullable: true},
|
||||||
{Name: "location_name", Type: "TEXT", Nullable: true},
|
{Name: "location_name", Type: "TEXT", Nullable: true},
|
||||||
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: false},
|
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: false},
|
||||||
@@ -89,7 +78,7 @@ func PostgresSchema() fksinks.PostgresSchema {
|
|||||||
{Name: "longitude", Type: "DOUBLE PRECISION", Nullable: true},
|
{Name: "longitude", Type: "DOUBLE PRECISION", Nullable: true},
|
||||||
{Name: "elevation_meters", Type: "DOUBLE PRECISION", Nullable: true},
|
{Name: "elevation_meters", Type: "DOUBLE PRECISION", Nullable: true},
|
||||||
{Name: "period_count", Type: "INTEGER", Nullable: false},
|
{Name: "period_count", Type: "INTEGER", Nullable: false},
|
||||||
},
|
}...),
|
||||||
PrimaryKey: []string{"event_id"},
|
PrimaryKey: []string{"event_id"},
|
||||||
PruneColumn: "issued_at",
|
PruneColumn: "issued_at",
|
||||||
Indexes: []fksinks.PostgresIndex{
|
Indexes: []fksinks.PostgresIndex{
|
||||||
@@ -137,13 +126,7 @@ func PostgresSchema() fksinks.PostgresSchema {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
Name: tableForecastDiscussions,
|
Name: tableForecastDiscussions,
|
||||||
Columns: []fksinks.PostgresColumn{
|
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
|
||||||
{Name: "event_id", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_kind", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_source", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_schema", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
|
|
||||||
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
|
|
||||||
{Name: "office_id", Type: "TEXT", Nullable: true},
|
{Name: "office_id", Type: "TEXT", Nullable: true},
|
||||||
{Name: "office_name", Type: "TEXT", Nullable: true},
|
{Name: "office_name", Type: "TEXT", Nullable: true},
|
||||||
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: false},
|
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: false},
|
||||||
@@ -156,7 +139,7 @@ func PostgresSchema() fksinks.PostgresSchema {
|
|||||||
{Name: "long_term_issued_at", Type: "TIMESTAMPTZ", Nullable: true},
|
{Name: "long_term_issued_at", Type: "TIMESTAMPTZ", Nullable: true},
|
||||||
{Name: "long_term_text", Type: "TEXT", Nullable: true},
|
{Name: "long_term_text", Type: "TEXT", Nullable: true},
|
||||||
{Name: "key_message_count", Type: "INTEGER", Nullable: false},
|
{Name: "key_message_count", Type: "INTEGER", Nullable: false},
|
||||||
},
|
}...),
|
||||||
PrimaryKey: []string{"event_id"},
|
PrimaryKey: []string{"event_id"},
|
||||||
PruneColumn: "issued_at",
|
PruneColumn: "issued_at",
|
||||||
Indexes: []fksinks.PostgresIndex{
|
Indexes: []fksinks.PostgresIndex{
|
||||||
@@ -180,17 +163,11 @@ func PostgresSchema() fksinks.PostgresSchema {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
Name: tableWeatherStoryRuns,
|
Name: tableWeatherStoryRuns,
|
||||||
Columns: []fksinks.PostgresColumn{
|
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
|
||||||
{Name: "event_id", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_kind", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_source", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_schema", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
|
|
||||||
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
|
|
||||||
{Name: "office_id", Type: "TEXT", Nullable: true},
|
{Name: "office_id", Type: "TEXT", Nullable: true},
|
||||||
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
|
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
|
||||||
{Name: "story_count", Type: "INTEGER", Nullable: false},
|
{Name: "story_count", Type: "INTEGER", Nullable: false},
|
||||||
},
|
}...),
|
||||||
PrimaryKey: []string{"event_id"},
|
PrimaryKey: []string{"event_id"},
|
||||||
PruneColumn: "as_of",
|
PruneColumn: "as_of",
|
||||||
Indexes: []fksinks.PostgresIndex{
|
Indexes: []fksinks.PostgresIndex{
|
||||||
@@ -225,20 +202,14 @@ func PostgresSchema() fksinks.PostgresSchema {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
Name: tableAlertRuns,
|
Name: tableAlertRuns,
|
||||||
Columns: []fksinks.PostgresColumn{
|
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
|
||||||
{Name: "event_id", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_kind", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_source", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_schema", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
|
|
||||||
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
|
|
||||||
{Name: "location_id", Type: "TEXT", Nullable: true},
|
{Name: "location_id", Type: "TEXT", Nullable: true},
|
||||||
{Name: "location_name", Type: "TEXT", Nullable: true},
|
{Name: "location_name", Type: "TEXT", Nullable: true},
|
||||||
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
|
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
|
||||||
{Name: "latitude", Type: "DOUBLE PRECISION", Nullable: true},
|
{Name: "latitude", Type: "DOUBLE PRECISION", Nullable: true},
|
||||||
{Name: "longitude", Type: "DOUBLE PRECISION", Nullable: true},
|
{Name: "longitude", Type: "DOUBLE PRECISION", Nullable: true},
|
||||||
{Name: "alert_count", Type: "INTEGER", Nullable: false},
|
{Name: "alert_count", Type: "INTEGER", Nullable: false},
|
||||||
},
|
}...),
|
||||||
PrimaryKey: []string{"event_id"},
|
PrimaryKey: []string{"event_id"},
|
||||||
PruneColumn: "as_of",
|
PruneColumn: "as_of",
|
||||||
Indexes: []fksinks.PostgresIndex{
|
Indexes: []fksinks.PostgresIndex{
|
||||||
@@ -301,13 +272,7 @@ func PostgresSchema() fksinks.PostgresSchema {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
Name: tableOutlookRuns,
|
Name: tableOutlookRuns,
|
||||||
Columns: []fksinks.PostgresColumn{
|
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
|
||||||
{Name: "event_id", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_kind", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_source", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_schema", Type: "TEXT", Nullable: false},
|
|
||||||
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
|
|
||||||
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
|
|
||||||
{Name: "location_id", Type: "TEXT", Nullable: true},
|
{Name: "location_id", Type: "TEXT", Nullable: true},
|
||||||
{Name: "location_name", Type: "TEXT", Nullable: true},
|
{Name: "location_name", Type: "TEXT", Nullable: true},
|
||||||
{Name: "latitude", Type: "DOUBLE PRECISION", Nullable: true},
|
{Name: "latitude", Type: "DOUBLE PRECISION", Nullable: true},
|
||||||
@@ -315,7 +280,8 @@ func PostgresSchema() fksinks.PostgresSchema {
|
|||||||
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
|
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
|
||||||
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: true},
|
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: true},
|
||||||
{Name: "outlook_count", Type: "INTEGER", Nullable: false},
|
{Name: "outlook_count", Type: "INTEGER", Nullable: false},
|
||||||
},
|
{Name: "discussion_count", Type: "INTEGER", Nullable: false},
|
||||||
|
}...),
|
||||||
PrimaryKey: []string{"event_id"},
|
PrimaryKey: []string{"event_id"},
|
||||||
PruneColumn: "as_of",
|
PruneColumn: "as_of",
|
||||||
Indexes: []fksinks.PostgresIndex{
|
Indexes: []fksinks.PostgresIndex{
|
||||||
@@ -342,9 +308,6 @@ func PostgresSchema() fksinks.PostgresSchema {
|
|||||||
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: false},
|
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: false},
|
||||||
{Name: "expires_at", Type: "TIMESTAMPTZ", Nullable: false},
|
{Name: "expires_at", Type: "TIMESTAMPTZ", Nullable: false},
|
||||||
{Name: "forecaster", Type: "TEXT", Nullable: true},
|
{Name: "forecaster", Type: "TEXT", Nullable: true},
|
||||||
{Name: "headline", Type: "TEXT", Nullable: true},
|
|
||||||
{Name: "summary", Type: "TEXT", Nullable: true},
|
|
||||||
{Name: "discussion", Type: "TEXT", Nullable: true},
|
|
||||||
{Name: "source_url", Type: "TEXT", Nullable: true},
|
{Name: "source_url", Type: "TEXT", Nullable: true},
|
||||||
{Name: "image_url", Type: "TEXT", Nullable: true},
|
{Name: "image_url", Type: "TEXT", Nullable: true},
|
||||||
{Name: "contains_location", Type: "BOOLEAN", Nullable: false},
|
{Name: "contains_location", Type: "BOOLEAN", Nullable: false},
|
||||||
@@ -358,7 +321,38 @@ func PostgresSchema() fksinks.PostgresSchema {
|
|||||||
{Name: "idx_wf_outlooks_valid", Columns: []string{"valid_from", "valid_to"}},
|
{Name: "idx_wf_outlooks_valid", Columns: []string{"valid_from", "valid_to"}},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
Name: tableOutlookDiscussions,
|
||||||
|
Columns: []fksinks.PostgresColumn{
|
||||||
|
{Name: "run_event_id", Type: "TEXT REFERENCES outlook_runs(event_id) ON DELETE CASCADE", Nullable: false},
|
||||||
|
{Name: "discussion_index", Type: "INTEGER", Nullable: false},
|
||||||
|
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
|
||||||
|
{Name: "day", Type: "INTEGER", Nullable: false},
|
||||||
|
{Name: "headline", Type: "TEXT", Nullable: true},
|
||||||
|
{Name: "summary", Type: "TEXT", Nullable: true},
|
||||||
|
{Name: "discussion", Type: "TEXT", Nullable: true},
|
||||||
|
{Name: "updated_at", Type: "TIMESTAMPTZ", Nullable: true},
|
||||||
|
},
|
||||||
|
PrimaryKey: []string{"run_event_id", "discussion_index"},
|
||||||
|
PruneColumn: "as_of",
|
||||||
|
Indexes: []fksinks.PostgresIndex{
|
||||||
|
{Name: "idx_wf_outlook_discussions_day_as_of", Columns: []string{"day", "as_of"}},
|
||||||
|
{Name: "idx_wf_outlook_discussions_run_day", Columns: []string{"run_event_id", "day"}, Unique: true},
|
||||||
|
},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
MapEvent: mapPostgresEvent,
|
MapEvent: mapPostgresEvent,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func parentEnvelopeColumns(extra ...fksinks.PostgresColumn) []fksinks.PostgresColumn {
|
||||||
|
columns := []fksinks.PostgresColumn{
|
||||||
|
{Name: "event_id", Type: "TEXT", Nullable: false},
|
||||||
|
{Name: "event_kind", Type: "TEXT", Nullable: false},
|
||||||
|
{Name: "event_source", Type: "TEXT", Nullable: false},
|
||||||
|
{Name: "event_schema", Type: "TEXT", Nullable: false},
|
||||||
|
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
|
||||||
|
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
|
||||||
|
}
|
||||||
|
return append(columns, extra...)
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,8 +1,11 @@
|
|||||||
package postgres
|
package postgres
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
fksinks "gitea.maximumdirect.net/ejr/feedkit/sinks"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestWeatherPostgresSchemaShape(t *testing.T) {
|
func TestWeatherPostgresSchemaShape(t *testing.T) {
|
||||||
@@ -25,6 +28,7 @@ func TestWeatherPostgresSchemaShape(t *testing.T) {
|
|||||||
tableAlertReferences: true,
|
tableAlertReferences: true,
|
||||||
tableOutlookRuns: true,
|
tableOutlookRuns: true,
|
||||||
tableOutlooks: true,
|
tableOutlooks: true,
|
||||||
|
tableOutlookDiscussions: true,
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(s.Tables) != len(wantTables) {
|
if len(s.Tables) != len(wantTables) {
|
||||||
@@ -50,7 +54,7 @@ func TestWeatherPostgresSchemaShape(t *testing.T) {
|
|||||||
|
|
||||||
func TestWeatherPostgresSchemaIncludesOutlookTables(t *testing.T) {
|
func TestWeatherPostgresSchemaIncludesOutlookTables(t *testing.T) {
|
||||||
runColumns := columnsForTable(t, tableOutlookRuns)
|
runColumns := columnsForTable(t, tableOutlookRuns)
|
||||||
for _, col := range []string{"event_id", "event_kind", "event_source", "event_schema", "event_emitted_at", "event_effective_at", "location_id", "location_name", "latitude", "longitude", "as_of", "issued_at", "outlook_count"} {
|
for _, col := range []string{"event_id", "event_kind", "event_source", "event_schema", "event_emitted_at", "event_effective_at", "location_id", "location_name", "latitude", "longitude", "as_of", "issued_at", "outlook_count", "discussion_count"} {
|
||||||
if !runColumns[col] {
|
if !runColumns[col] {
|
||||||
t.Fatalf("%s missing %s column", tableOutlookRuns, col)
|
t.Fatalf("%s missing %s column", tableOutlookRuns, col)
|
||||||
}
|
}
|
||||||
@@ -60,15 +64,31 @@ func TestWeatherPostgresSchemaIncludesOutlookTables(t *testing.T) {
|
|||||||
assertTableIndex(t, tableOutlookRuns, "idx_wf_outlook_run_as_of", []string{"as_of"})
|
assertTableIndex(t, tableOutlookRuns, "idx_wf_outlook_run_as_of", []string{"as_of"})
|
||||||
|
|
||||||
outlookColumns := columnsForTable(t, tableOutlooks)
|
outlookColumns := columnsForTable(t, tableOutlooks)
|
||||||
for _, col := range []string{"run_event_id", "outlook_index", "as_of", "outlook_id", "provider", "product", "day", "outlook_type", "label", "label_text", "severity_rank", "valid_from", "valid_to", "issued_at", "expires_at", "forecaster", "headline", "summary", "discussion", "source_url", "image_url", "contains_location", "geometry_json"} {
|
for _, col := range []string{"run_event_id", "outlook_index", "as_of", "outlook_id", "provider", "product", "day", "outlook_type", "label", "label_text", "severity_rank", "valid_from", "valid_to", "issued_at", "expires_at", "forecaster", "source_url", "image_url", "contains_location", "geometry_json"} {
|
||||||
if !outlookColumns[col] {
|
if !outlookColumns[col] {
|
||||||
t.Fatalf("%s missing %s column", tableOutlooks, col)
|
t.Fatalf("%s missing %s column", tableOutlooks, col)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
for _, col := range []string{"headline", "summary", "discussion"} {
|
||||||
|
if outlookColumns[col] {
|
||||||
|
t.Fatalf("%s still includes legacy %s column", tableOutlooks, col)
|
||||||
|
}
|
||||||
|
}
|
||||||
assertTablePrimaryKey(t, tableOutlooks, []string{"run_event_id", "outlook_index"})
|
assertTablePrimaryKey(t, tableOutlooks, []string{"run_event_id", "outlook_index"})
|
||||||
assertTableIndex(t, tableOutlooks, "idx_wf_outlooks_contains_valid", []string{"contains_location", "valid_from", "valid_to"})
|
assertTableIndex(t, tableOutlooks, "idx_wf_outlooks_contains_valid", []string{"contains_location", "valid_from", "valid_to"})
|
||||||
assertTableIndex(t, tableOutlooks, "idx_wf_outlooks_day_type_label", []string{"day", "outlook_type", "label"})
|
assertTableIndex(t, tableOutlooks, "idx_wf_outlooks_day_type_label", []string{"day", "outlook_type", "label"})
|
||||||
assertTableIndex(t, tableOutlooks, "idx_wf_outlooks_valid", []string{"valid_from", "valid_to"})
|
assertTableIndex(t, tableOutlooks, "idx_wf_outlooks_valid", []string{"valid_from", "valid_to"})
|
||||||
|
|
||||||
|
discussionColumns := columnsForTable(t, tableOutlookDiscussions)
|
||||||
|
for _, col := range []string{"run_event_id", "discussion_index", "as_of", "day", "headline", "summary", "discussion", "updated_at"} {
|
||||||
|
if !discussionColumns[col] {
|
||||||
|
t.Fatalf("%s missing %s column", tableOutlookDiscussions, col)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assertTablePrimaryKey(t, tableOutlookDiscussions, []string{"run_event_id", "discussion_index"})
|
||||||
|
assertTablePruneColumn(t, tableOutlookDiscussions, "as_of")
|
||||||
|
assertTableIndex(t, tableOutlookDiscussions, "idx_wf_outlook_discussions_day_as_of", []string{"day", "as_of"})
|
||||||
|
assertTableUniqueIndex(t, tableOutlookDiscussions, "idx_wf_outlook_discussions_run_day", []string{"run_event_id", "day"})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWeatherPostgresSchemaIncludesWeatherStoryColumns(t *testing.T) {
|
func TestWeatherPostgresSchemaIncludesWeatherStoryColumns(t *testing.T) {
|
||||||
@@ -88,53 +108,102 @@ func TestWeatherPostgresSchemaIncludesWeatherStoryColumns(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestWeatherPostgresSchemaParentTablesStartWithEnvelopeColumns(t *testing.T) {
|
||||||
|
for _, table := range []string{
|
||||||
|
tableObservations,
|
||||||
|
tableForecasts,
|
||||||
|
tableForecastDiscussions,
|
||||||
|
tableWeatherStoryRuns,
|
||||||
|
tableAlertRuns,
|
||||||
|
tableOutlookRuns,
|
||||||
|
} {
|
||||||
|
t.Run(table, func(t *testing.T) {
|
||||||
|
columns := orderedColumnsForTable(t, table)
|
||||||
|
want := parentEnvelopeColumns()
|
||||||
|
if len(columns) < len(want) {
|
||||||
|
t.Fatalf("%s has %d columns, want at least %d", table, len(columns), len(want))
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(columns[:len(want)], want) {
|
||||||
|
t.Fatalf("%s envelope prefix = %#v, want %#v", table, columns[:len(want)], want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func assertTablePrimaryKey(t *testing.T, table string, want []string) {
|
func assertTablePrimaryKey(t *testing.T, table string, want []string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
for _, tbl := range PostgresSchema().Tables {
|
tbl := tableByName(t, table)
|
||||||
if tbl.Name != table {
|
if strings.Join(tbl.PrimaryKey, ",") != strings.Join(want, ",") {
|
||||||
continue
|
t.Fatalf("%s primary key = %#v, want %#v", table, tbl.PrimaryKey, want)
|
||||||
}
|
}
|
||||||
if strings.Join(tbl.PrimaryKey, ",") != strings.Join(want, ",") {
|
}
|
||||||
t.Fatalf("%s primary key = %#v, want %#v", table, tbl.PrimaryKey, want)
|
|
||||||
}
|
func assertTablePruneColumn(t *testing.T, table string, want string) {
|
||||||
return
|
t.Helper()
|
||||||
|
tbl := tableByName(t, table)
|
||||||
|
if tbl.PruneColumn != want {
|
||||||
|
t.Fatalf("%s prune column = %q, want %q", table, tbl.PruneColumn, want)
|
||||||
}
|
}
|
||||||
t.Fatalf("missing table %q", table)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func assertTableIndex(t *testing.T, table string, name string, want []string) {
|
func assertTableIndex(t *testing.T, table string, name string, want []string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
for _, tbl := range PostgresSchema().Tables {
|
assertTableIndexWithUnique(t, table, name, want, false)
|
||||||
if tbl.Name != table {
|
}
|
||||||
continue
|
|
||||||
}
|
func assertTableUniqueIndex(t *testing.T, table string, name string, want []string) {
|
||||||
for _, idx := range tbl.Indexes {
|
t.Helper()
|
||||||
if idx.Name == name {
|
assertTableIndexWithUnique(t, table, name, want, true)
|
||||||
if strings.Join(idx.Columns, ",") != strings.Join(want, ",") {
|
}
|
||||||
t.Fatalf("%s index %s columns = %#v, want %#v", table, name, idx.Columns, want)
|
|
||||||
}
|
func assertTableIndexWithUnique(t *testing.T, table string, name string, want []string, unique bool) {
|
||||||
return
|
t.Helper()
|
||||||
|
tbl := tableByName(t, table)
|
||||||
|
for _, idx := range tbl.Indexes {
|
||||||
|
if idx.Name == name {
|
||||||
|
if strings.Join(idx.Columns, ",") != strings.Join(want, ",") {
|
||||||
|
t.Fatalf("%s index %s columns = %#v, want %#v", table, name, idx.Columns, want)
|
||||||
}
|
}
|
||||||
|
if idx.Unique != unique {
|
||||||
|
t.Fatalf("%s index %s unique = %v, want %v", table, name, idx.Unique, unique)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("%s missing index %s", table, name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func tableByName(t *testing.T, table string) fksinks.PostgresTable {
|
||||||
|
t.Helper()
|
||||||
|
for _, tbl := range PostgresSchema().Tables {
|
||||||
|
if tbl.Name == table {
|
||||||
|
return tbl
|
||||||
}
|
}
|
||||||
t.Fatalf("%s missing index %s", table, name)
|
|
||||||
}
|
}
|
||||||
t.Fatalf("missing table %q", table)
|
t.Fatalf("missing table %q", table)
|
||||||
|
return fksinks.PostgresTable{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func orderedColumnsForTable(t *testing.T, table string) []fksinks.PostgresColumn {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
schema := PostgresSchema()
|
||||||
|
for _, tbl := range schema.Tables {
|
||||||
|
if tbl.Name == table {
|
||||||
|
return tbl.Columns
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("missing table %q", table)
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func columnsForTable(t *testing.T, table string) map[string]bool {
|
func columnsForTable(t *testing.T, table string) map[string]bool {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
schema := PostgresSchema()
|
ordered := orderedColumnsForTable(t, table)
|
||||||
for _, tbl := range schema.Tables {
|
cols := make(map[string]bool, len(ordered))
|
||||||
if tbl.Name != table {
|
for _, col := range ordered {
|
||||||
continue
|
cols[col.Name] = true
|
||||||
}
|
|
||||||
cols := make(map[string]bool, len(tbl.Columns))
|
|
||||||
for _, col := range tbl.Columns {
|
|
||||||
cols[col.Name] = true
|
|
||||||
}
|
|
||||||
return cols
|
|
||||||
}
|
}
|
||||||
t.Fatalf("missing table %q", table)
|
return cols
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,20 +16,20 @@ type pollDriverRegistration struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var pollDriverRegistrations = []pollDriverRegistration{
|
var pollDriverRegistrations = []pollDriverRegistration{
|
||||||
{driver: "nws_observation", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewObservationSource(cfg) }},
|
{driver: nws.DriverObservation, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewObservationSource(cfg) }},
|
||||||
{driver: "nws_alerts", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewAlertsSource(cfg) }},
|
{driver: nws.DriverAlerts, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewAlertsSource(cfg) }},
|
||||||
{driver: "nws_forecast_hourly", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewHourlyForecastSource(cfg) }},
|
{driver: nws.DriverForecastHourly, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewHourlyForecastSource(cfg) }},
|
||||||
{driver: "nws_forecast_narrative", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewNarrativeForecastSource(cfg) }},
|
{driver: nws.DriverForecastNarrative, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewNarrativeForecastSource(cfg) }},
|
||||||
{driver: "nws_forecast_discussion", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
|
{driver: nws.DriverForecastDiscussion, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
|
||||||
return nws.NewForecastDiscussionSource(cfg)
|
return nws.NewForecastDiscussionSource(cfg)
|
||||||
}},
|
}},
|
||||||
{driver: "nws_weatherstories", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewWeatherStoriesSource(cfg) }},
|
{driver: nws.DriverWeatherStories, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewWeatherStoriesSource(cfg) }},
|
||||||
{driver: "openmeteo_observation", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewObservationSource(cfg) }},
|
{driver: openmeteo.DriverObservation, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewObservationSource(cfg) }},
|
||||||
{driver: "openmeteo_forecast", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewForecastSource(cfg) }},
|
{driver: openmeteo.DriverForecast, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewForecastSource(cfg) }},
|
||||||
{driver: "openweather_observation", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
|
{driver: openweather.DriverObservation, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
|
||||||
return openweather.NewObservationSource(cfg)
|
return openweather.NewObservationSource(cfg)
|
||||||
}},
|
}},
|
||||||
{driver: "spc_convective_outlook", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
|
{driver: spc.DriverConvectiveOutlook, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
|
||||||
return spc.NewConvectiveOutlookSource(cfg)
|
return spc.NewConvectiveOutlookSource(cfg)
|
||||||
}},
|
}},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,57 +6,29 @@ import (
|
|||||||
|
|
||||||
"gitea.maximumdirect.net/ejr/feedkit/config"
|
"gitea.maximumdirect.net/ejr/feedkit/config"
|
||||||
fksource "gitea.maximumdirect.net/ejr/feedkit/sources"
|
fksource "gitea.maximumdirect.net/ejr/feedkit/sources"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/openweather"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/spc"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestRegisterBuiltinsRegistersNWSHourlyForecastDriver(t *testing.T) {
|
func TestRegisterBuiltinsRegistersCurrentPollDrivers(t *testing.T) {
|
||||||
reg := fksource.NewRegistry()
|
reg := fksource.NewRegistry()
|
||||||
RegisterBuiltins(reg)
|
RegisterBuiltins(reg)
|
||||||
|
|
||||||
in, err := reg.BuildInput(sourceConfigForDriver("nws_forecast_hourly"))
|
if len(pollDriverRegistrations) == 0 {
|
||||||
if err != nil {
|
t.Fatalf("pollDriverRegistrations is empty")
|
||||||
t.Fatalf("BuildInput(nws_forecast_hourly) error = %v", err)
|
|
||||||
}
|
}
|
||||||
if _, ok := in.(fksource.PollSource); !ok {
|
|
||||||
t.Fatalf("BuildInput(nws_forecast_hourly) type = %T, want PollSource", in)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRegisterBuiltinsRegistersNWSNarrativeForecastDriver(t *testing.T) {
|
for _, tt := range pollDriverRegistrations {
|
||||||
reg := fksource.NewRegistry()
|
tt := tt
|
||||||
RegisterBuiltins(reg)
|
t.Run(tt.driver, func(t *testing.T) {
|
||||||
|
in, err := reg.BuildInput(sourceConfigForDriver(tt.driver))
|
||||||
in, err := reg.BuildInput(sourceConfigForDriver("nws_forecast_narrative"))
|
if err != nil {
|
||||||
if err != nil {
|
t.Fatalf("BuildInput(%q) error = %v", tt.driver, err)
|
||||||
t.Fatalf("BuildInput(nws_forecast_narrative) error = %v", err)
|
}
|
||||||
}
|
if _, ok := in.(fksource.PollSource); !ok {
|
||||||
if _, ok := in.(fksource.PollSource); !ok {
|
t.Fatalf("BuildInput(%q) type = %T, want PollSource", tt.driver, in)
|
||||||
t.Fatalf("BuildInput(nws_forecast_narrative) type = %T, want PollSource", in)
|
}
|
||||||
}
|
})
|
||||||
}
|
|
||||||
|
|
||||||
func TestRegisterBuiltinsRegistersNWSForecastDiscussionDriver(t *testing.T) {
|
|
||||||
reg := fksource.NewRegistry()
|
|
||||||
RegisterBuiltins(reg)
|
|
||||||
|
|
||||||
in, err := reg.BuildInput(sourceConfigForDriver("nws_forecast_discussion"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("BuildInput(nws_forecast_discussion) error = %v", err)
|
|
||||||
}
|
|
||||||
if _, ok := in.(fksource.PollSource); !ok {
|
|
||||||
t.Fatalf("BuildInput(nws_forecast_discussion) type = %T, want PollSource", in)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRegisterBuiltinsRegistersNWSWeatherStoriesDriver(t *testing.T) {
|
|
||||||
reg := fksource.NewRegistry()
|
|
||||||
RegisterBuiltins(reg)
|
|
||||||
|
|
||||||
in, err := reg.BuildInput(sourceConfigForDriver("nws_weatherstories"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("BuildInput(nws_weatherstories) error = %v", err)
|
|
||||||
}
|
|
||||||
if _, ok := in.(fksource.PollSource); !ok {
|
|
||||||
t.Fatalf("BuildInput(nws_weatherstories) type = %T, want PollSource", in)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -73,44 +45,16 @@ func TestRegisterBuiltinsDoesNotRegisterLegacyNWSForecastDriver(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRegisterBuiltinsRegistersAllCurrentDrivers(t *testing.T) {
|
|
||||||
reg := fksource.NewRegistry()
|
|
||||||
RegisterBuiltins(reg)
|
|
||||||
|
|
||||||
drivers := []string{
|
|
||||||
"nws_observation",
|
|
||||||
"nws_alerts",
|
|
||||||
"nws_forecast_hourly",
|
|
||||||
"nws_forecast_narrative",
|
|
||||||
"nws_forecast_discussion",
|
|
||||||
"nws_weatherstories",
|
|
||||||
"openmeteo_observation",
|
|
||||||
"openmeteo_forecast",
|
|
||||||
"openweather_observation",
|
|
||||||
"spc_convective_outlook",
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, driver := range drivers {
|
|
||||||
in, err := reg.BuildInput(sourceConfigForDriver(driver))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("BuildInput(%s) error = %v", driver, err)
|
|
||||||
}
|
|
||||||
if _, ok := in.(fksource.PollSource); !ok {
|
|
||||||
t.Fatalf("BuildInput(%s) type = %T, want PollSource", driver, in)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func sourceConfigForDriver(driver string) config.SourceConfig {
|
func sourceConfigForDriver(driver string) config.SourceConfig {
|
||||||
url := "https://example.invalid"
|
url := "https://example.invalid"
|
||||||
if driver == "openweather_observation" {
|
if driver == openweather.DriverObservation {
|
||||||
url = "https://example.invalid?units=metric"
|
url = "https://example.invalid?units=metric"
|
||||||
}
|
}
|
||||||
params := map[string]any{
|
params := map[string]any{
|
||||||
"url": url,
|
"url": url,
|
||||||
"user_agent": "test-agent",
|
"user_agent": "test-agent",
|
||||||
}
|
}
|
||||||
if driver == "spc_convective_outlook" {
|
if driver == spc.DriverConvectiveOutlook {
|
||||||
params["latitude"] = 38.6239
|
params["latitude"] = 38.6239
|
||||||
params["longitude"] = -90.3571
|
params["longitude"] = -90.3571
|
||||||
}
|
}
|
||||||
|
|||||||
32
internal/sources/docs_test.go
Normal file
32
internal/sources/docs_test.go
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
package sources
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDocumentedRegisteredSourceDrivers(t *testing.T) {
|
||||||
|
docs := map[string]string{
|
||||||
|
"docs/config.md": readDoc(t, "../../docs/config.md"),
|
||||||
|
"docs/internal/sources.md": readDoc(t, "../../docs/internal/sources.md"),
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, reg := range pollDriverRegistrations {
|
||||||
|
for path, doc := range docs {
|
||||||
|
if !strings.Contains(doc, reg.driver) {
|
||||||
|
t.Fatalf("%s missing source driver %q", path, reg.driver)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func readDoc(t *testing.T, path string) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile(%s) error = %v", path, err)
|
||||||
|
}
|
||||||
|
return string(raw)
|
||||||
|
}
|
||||||
58
internal/sources/internal/httpconfig/config.go
Normal file
58
internal/sources/internal/httpconfig/config.go
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
package httpconfig
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/ejr/feedkit/config"
|
||||||
|
"gitea.maximumdirect.net/ejr/feedkit/transport"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Settings contains common HTTP client config for sources that fetch multiple documents.
|
||||||
|
type Settings struct {
|
||||||
|
Name string
|
||||||
|
UserAgent string
|
||||||
|
Timeout time.Duration
|
||||||
|
BodyLimitBytes int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func Parse(driver string, cfg config.SourceConfig) (Settings, error) {
|
||||||
|
name := strings.TrimSpace(cfg.Name)
|
||||||
|
if name == "" {
|
||||||
|
return Settings{}, fmt.Errorf("%s: name is required", driver)
|
||||||
|
}
|
||||||
|
if cfg.Params == nil {
|
||||||
|
return Settings{}, fmt.Errorf("%s %q: params are required", driver, name)
|
||||||
|
}
|
||||||
|
|
||||||
|
userAgent, ok := cfg.ParamString("user_agent", "userAgent")
|
||||||
|
if !ok {
|
||||||
|
return Settings{}, fmt.Errorf("%s %q: params.user_agent is required", driver, name)
|
||||||
|
}
|
||||||
|
|
||||||
|
timeout := transport.DefaultHTTPTimeout
|
||||||
|
if _, exists := cfg.Params["http_timeout"]; exists {
|
||||||
|
var ok bool
|
||||||
|
timeout, ok = cfg.ParamDuration("http_timeout")
|
||||||
|
if !ok || timeout <= 0 {
|
||||||
|
return Settings{}, fmt.Errorf("source %q: params.http_timeout must be a positive duration", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bodyLimit := transport.DefaultHTTPResponseBodyLimitBytes
|
||||||
|
if _, exists := cfg.Params["http_response_body_limit_bytes"]; exists {
|
||||||
|
rawLimit, ok := cfg.ParamInt("http_response_body_limit_bytes")
|
||||||
|
if !ok || rawLimit <= 0 {
|
||||||
|
return Settings{}, fmt.Errorf("source %q: params.http_response_body_limit_bytes must be a positive integer", name)
|
||||||
|
}
|
||||||
|
bodyLimit = int64(rawLimit)
|
||||||
|
}
|
||||||
|
|
||||||
|
return Settings{
|
||||||
|
Name: name,
|
||||||
|
UserAgent: userAgent,
|
||||||
|
Timeout: timeout,
|
||||||
|
BodyLimitBytes: bodyLimit,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
115
internal/sources/internal/httpconfig/config_test.go
Normal file
115
internal/sources/internal/httpconfig/config_test.go
Normal file
@@ -0,0 +1,115 @@
|
|||||||
|
package httpconfig
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/ejr/feedkit/config"
|
||||||
|
"gitea.maximumdirect.net/ejr/feedkit/transport"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseUsesRequiredValuesAndDefaults(t *testing.T) {
|
||||||
|
got, err := Parse("test_driver", config.SourceConfig{
|
||||||
|
Name: " test-source ",
|
||||||
|
Params: map[string]any{
|
||||||
|
"user_agent": "test-agent",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Parse() error = %v", err)
|
||||||
|
}
|
||||||
|
if got.Name != "test-source" {
|
||||||
|
t.Fatalf("Name = %q, want test-source", got.Name)
|
||||||
|
}
|
||||||
|
if got.UserAgent != "test-agent" {
|
||||||
|
t.Fatalf("UserAgent = %q, want test-agent", got.UserAgent)
|
||||||
|
}
|
||||||
|
if got.Timeout != transport.DefaultHTTPTimeout {
|
||||||
|
t.Fatalf("Timeout = %s, want %s", got.Timeout, transport.DefaultHTTPTimeout)
|
||||||
|
}
|
||||||
|
if got.BodyLimitBytes != transport.DefaultHTTPResponseBodyLimitBytes {
|
||||||
|
t.Fatalf("BodyLimitBytes = %d, want %d", got.BodyLimitBytes, transport.DefaultHTTPResponseBodyLimitBytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseUsesAliasesAndOverrides(t *testing.T) {
|
||||||
|
got, err := Parse("test_driver", config.SourceConfig{
|
||||||
|
Name: "test-source",
|
||||||
|
Params: map[string]any{
|
||||||
|
"userAgent": "test-agent",
|
||||||
|
"http_timeout": "2s",
|
||||||
|
"http_response_body_limit_bytes": 2048,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Parse() error = %v", err)
|
||||||
|
}
|
||||||
|
if got.UserAgent != "test-agent" {
|
||||||
|
t.Fatalf("UserAgent = %q, want test-agent", got.UserAgent)
|
||||||
|
}
|
||||||
|
if got.Timeout != 2*time.Second {
|
||||||
|
t.Fatalf("Timeout = %s, want 2s", got.Timeout)
|
||||||
|
}
|
||||||
|
if got.BodyLimitBytes != 2048 {
|
||||||
|
t.Fatalf("BodyLimitBytes = %d, want 2048", got.BodyLimitBytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseRejectsInvalidConfig(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
cfg config.SourceConfig
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "missing name",
|
||||||
|
cfg: config.SourceConfig{Params: map[string]any{"user_agent": "test-agent"}},
|
||||||
|
wantErr: "test_driver: name is required",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing params",
|
||||||
|
cfg: config.SourceConfig{Name: "test-source"},
|
||||||
|
wantErr: `test_driver "test-source": params are required`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing user agent",
|
||||||
|
cfg: config.SourceConfig{Name: "test-source", Params: map[string]any{}},
|
||||||
|
wantErr: `test_driver "test-source": params.user_agent is required`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid timeout",
|
||||||
|
cfg: config.SourceConfig{
|
||||||
|
Name: "test-source",
|
||||||
|
Params: map[string]any{
|
||||||
|
"user_agent": "test-agent",
|
||||||
|
"http_timeout": "0s",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
wantErr: `source "test-source": params.http_timeout must be a positive duration`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid body limit",
|
||||||
|
cfg: config.SourceConfig{
|
||||||
|
Name: "test-source",
|
||||||
|
Params: map[string]any{
|
||||||
|
"user_agent": "test-agent",
|
||||||
|
"http_response_body_limit_bytes": 0,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
wantErr: `source "test-source": params.http_response_body_limit_bytes must be a positive integer`,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
_, err := Parse("test_driver", tt.cfg)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("Parse() error = nil, want %q", tt.wantErr)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), tt.wantErr) {
|
||||||
|
t.Fatalf("Parse() error = %q, want %q", err, tt.wantErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -26,10 +26,8 @@ type AlertsSource struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewAlertsSource(cfg config.SourceConfig) (*AlertsSource, error) {
|
func NewAlertsSource(cfg config.SourceConfig) (*AlertsSource, error) {
|
||||||
const driver = "nws_alerts"
|
|
||||||
|
|
||||||
// NWS alerts responses are GeoJSON-ish; allow fallback to plain JSON as well.
|
// NWS alerts responses are GeoJSON-ish; allow fallback to plain JSON as well.
|
||||||
hs, err := fksources.NewHTTPSource(driver, cfg, "application/geo+json, application/json")
|
hs, err := fksources.NewHTTPSource(DriverAlerts, cfg, "application/geo+json, application/json")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -40,7 +38,7 @@ func NewAlertsSource(cfg config.SourceConfig) (*AlertsSource, error) {
|
|||||||
func (s *AlertsSource) Name() string { return s.http.Name }
|
func (s *AlertsSource) Name() string { return s.http.Name }
|
||||||
|
|
||||||
// Kinds is used for routing/policy.
|
// Kinds is used for routing/policy.
|
||||||
func (s *AlertsSource) Kinds() []event.Kind { return []event.Kind{event.Kind("alert")} }
|
func (s *AlertsSource) Kinds() []event.Kind { return []event.Kind{event.Kind(standards.KindAlert)} }
|
||||||
|
|
||||||
func (s *AlertsSource) Poll(ctx context.Context) ([]event.Event, error) {
|
func (s *AlertsSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||||
raw, meta, changed, err := s.fetchRaw(ctx)
|
raw, meta, changed, err := s.fetchRaw(ctx)
|
||||||
@@ -71,7 +69,7 @@ func (s *AlertsSource) Poll(ctx context.Context) ([]event.Event, error) {
|
|||||||
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
||||||
|
|
||||||
return fksources.SingleEvent(
|
return fksources.SingleEvent(
|
||||||
event.Kind("alert"),
|
event.Kind(standards.KindAlert),
|
||||||
s.http.Name,
|
s.http.Name,
|
||||||
standards.SchemaRawNWSAlertsV1,
|
standards.SchemaRawNWSAlertsV1,
|
||||||
eventID,
|
eventID,
|
||||||
|
|||||||
11
internal/sources/nws/driver.go
Normal file
11
internal/sources/nws/driver.go
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
package nws
|
||||||
|
|
||||||
|
// Source driver strings registered by weatherfeeder for NWS sources.
|
||||||
|
const (
|
||||||
|
DriverObservation = "nws_observation"
|
||||||
|
DriverAlerts = "nws_alerts"
|
||||||
|
DriverForecastHourly = "nws_forecast_hourly"
|
||||||
|
DriverForecastNarrative = "nws_forecast_narrative"
|
||||||
|
DriverForecastDiscussion = "nws_forecast_discussion"
|
||||||
|
DriverWeatherStories = "nws_weatherstories"
|
||||||
|
)
|
||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/ejr/feedkit/event"
|
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||||
fksources "gitea.maximumdirect.net/ejr/feedkit/sources"
|
fksources "gitea.maximumdirect.net/ejr/feedkit/sources"
|
||||||
nwscommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/nws"
|
nwscommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/nws"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||||
)
|
)
|
||||||
|
|
||||||
const nwsForecastAccept = "application/geo+json, application/json"
|
const nwsForecastAccept = "application/geo+json, application/json"
|
||||||
@@ -44,7 +45,9 @@ func newForecastSource(cfg config.SourceConfig, driver, rawSchema string) (*fore
|
|||||||
|
|
||||||
func (s *forecastSource) Name() string { return s.http.Name }
|
func (s *forecastSource) Name() string { return s.http.Name }
|
||||||
|
|
||||||
func (s *forecastSource) Kinds() []event.Kind { return []event.Kind{event.Kind("forecast")} }
|
func (s *forecastSource) Kinds() []event.Kind {
|
||||||
|
return []event.Kind{event.Kind(standards.KindForecast)}
|
||||||
|
}
|
||||||
|
|
||||||
func (s *forecastSource) Poll(ctx context.Context) ([]event.Event, error) {
|
func (s *forecastSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||||
raw, meta, changed, err := s.fetchRaw(ctx)
|
raw, meta, changed, err := s.fetchRaw(ctx)
|
||||||
@@ -69,7 +72,7 @@ func (s *forecastSource) Poll(ctx context.Context) ([]event.Event, error) {
|
|||||||
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
||||||
|
|
||||||
return fksources.SingleEvent(
|
return fksources.SingleEvent(
|
||||||
event.Kind("forecast"),
|
event.Kind(standards.KindForecast),
|
||||||
s.http.Name,
|
s.http.Name,
|
||||||
s.rawSchema,
|
s.rawSchema,
|
||||||
eventID,
|
eventID,
|
||||||
|
|||||||
@@ -20,9 +20,7 @@ type ForecastDiscussionSource struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewForecastDiscussionSource(cfg config.SourceConfig) (*ForecastDiscussionSource, error) {
|
func NewForecastDiscussionSource(cfg config.SourceConfig) (*ForecastDiscussionSource, error) {
|
||||||
const driver = "nws_forecast_discussion"
|
hs, err := fksources.NewHTTPSource(DriverForecastDiscussion, cfg, "text/html, application/xhtml+xml")
|
||||||
|
|
||||||
hs, err := fksources.NewHTTPSource(driver, cfg, "text/html, application/xhtml+xml")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -33,7 +31,7 @@ func NewForecastDiscussionSource(cfg config.SourceConfig) (*ForecastDiscussionSo
|
|||||||
func (s *ForecastDiscussionSource) Name() string { return s.http.Name }
|
func (s *ForecastDiscussionSource) Name() string { return s.http.Name }
|
||||||
|
|
||||||
func (s *ForecastDiscussionSource) Kinds() []event.Kind {
|
func (s *ForecastDiscussionSource) Kinds() []event.Kind {
|
||||||
return []event.Kind{event.Kind("forecast_discussion")}
|
return []event.Kind{event.Kind(standards.KindForecastDiscussion)}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *ForecastDiscussionSource) Poll(ctx context.Context) ([]event.Event, error) {
|
func (s *ForecastDiscussionSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||||
@@ -57,7 +55,7 @@ func (s *ForecastDiscussionSource) Poll(ctx context.Context) ([]event.Event, err
|
|||||||
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
||||||
|
|
||||||
return fksources.SingleEvent(
|
return fksources.SingleEvent(
|
||||||
event.Kind("forecast_discussion"),
|
event.Kind(standards.KindForecastDiscussion),
|
||||||
s.http.Name,
|
s.http.Name,
|
||||||
standards.SchemaRawNWSForecastDiscussionV1,
|
standards.SchemaRawNWSForecastDiscussionV1,
|
||||||
eventID,
|
eventID,
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ func TestForecastDiscussionSourcePollEmitsExpectedEvent(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewForecastDiscussionSource() error = %v", err)
|
t.Fatalf("NewForecastDiscussionSource() error = %v", err)
|
||||||
}
|
}
|
||||||
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("forecast_discussion") {
|
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindForecastDiscussion) {
|
||||||
t.Fatalf("Kinds() = %#v, want [forecast_discussion]", got)
|
t.Fatalf("Kinds() = %#v, want [forecast_discussion]", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -40,7 +40,7 @@ func TestForecastDiscussionSourcePollEmitsExpectedEvent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
got := events[0]
|
got := events[0]
|
||||||
if got.Kind != event.Kind("forecast_discussion") {
|
if got.Kind != event.Kind(standards.KindForecastDiscussion) {
|
||||||
t.Fatalf("Kind = %q, want forecast_discussion", got.Kind)
|
t.Fatalf("Kind = %q, want forecast_discussion", got.Kind)
|
||||||
}
|
}
|
||||||
if got.Schema != standards.SchemaRawNWSForecastDiscussionV1 {
|
if got.Schema != standards.SchemaRawNWSForecastDiscussionV1 {
|
||||||
@@ -117,7 +117,7 @@ func TestForecastDiscussionSourcePollRejectsInvalidHTML(t *testing.T) {
|
|||||||
func forecastDiscussionSourceConfig(url string) config.SourceConfig {
|
func forecastDiscussionSourceConfig(url string) config.SourceConfig {
|
||||||
return config.SourceConfig{
|
return config.SourceConfig{
|
||||||
Name: "test-forecast-discussion-source",
|
Name: "test-forecast-discussion-source",
|
||||||
Driver: "nws_forecast_discussion",
|
Driver: DriverForecastDiscussion,
|
||||||
Mode: config.SourceModePoll,
|
Mode: config.SourceModePoll,
|
||||||
Params: map[string]any{
|
Params: map[string]any{
|
||||||
"url": url,
|
"url": url,
|
||||||
|
|||||||
@@ -18,8 +18,7 @@ type HourlyForecastSource struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewHourlyForecastSource(cfg config.SourceConfig) (*HourlyForecastSource, error) {
|
func NewHourlyForecastSource(cfg config.SourceConfig) (*HourlyForecastSource, error) {
|
||||||
const driver = "nws_forecast_hourly"
|
src, err := newForecastSource(cfg, DriverForecastHourly, standards.SchemaRawNWSHourlyForecastV1)
|
||||||
src, err := newForecastSource(cfg, driver, standards.SchemaRawNWSHourlyForecastV1)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,8 +18,7 @@ type NarrativeForecastSource struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewNarrativeForecastSource(cfg config.SourceConfig) (*NarrativeForecastSource, error) {
|
func NewNarrativeForecastSource(cfg config.SourceConfig) (*NarrativeForecastSource, error) {
|
||||||
const driver = "nws_forecast_narrative"
|
src, err := newForecastSource(cfg, DriverForecastNarrative, standards.SchemaRawNWSNarrativeForecastV1)
|
||||||
src, err := newForecastSource(cfg, driver, standards.SchemaRawNWSNarrativeForecastV1)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "hourly",
|
name: "hourly",
|
||||||
driver: "nws_forecast_hourly",
|
driver: DriverForecastHourly,
|
||||||
wantSchema: standards.SchemaRawNWSHourlyForecastV1,
|
wantSchema: standards.SchemaRawNWSHourlyForecastV1,
|
||||||
newSource: func(cfg config.SourceConfig) (forecastPoller, error) {
|
newSource: func(cfg config.SourceConfig) (forecastPoller, error) {
|
||||||
return NewHourlyForecastSource(cfg)
|
return NewHourlyForecastSource(cfg)
|
||||||
@@ -34,7 +34,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "narrative",
|
name: "narrative",
|
||||||
driver: "nws_forecast_narrative",
|
driver: DriverForecastNarrative,
|
||||||
wantSchema: standards.SchemaRawNWSNarrativeForecastV1,
|
wantSchema: standards.SchemaRawNWSNarrativeForecastV1,
|
||||||
newSource: func(cfg config.SourceConfig) (forecastPoller, error) {
|
newSource: func(cfg config.SourceConfig) (forecastPoller, error) {
|
||||||
return NewNarrativeForecastSource(cfg)
|
return NewNarrativeForecastSource(cfg)
|
||||||
@@ -55,7 +55,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
|
|||||||
}
|
}
|
||||||
if ks, ok := src.(interface{ Kinds() []event.Kind }); !ok {
|
if ks, ok := src.(interface{ Kinds() []event.Kind }); !ok {
|
||||||
t.Fatalf("source does not implement Kinds()")
|
t.Fatalf("source does not implement Kinds()")
|
||||||
} else if gotKinds := ks.Kinds(); len(gotKinds) != 1 || gotKinds[0] != event.Kind("forecast") {
|
} else if gotKinds := ks.Kinds(); len(gotKinds) != 1 || gotKinds[0] != event.Kind(standards.KindForecast) {
|
||||||
t.Fatalf("Kinds() = %#v, want [forecast]", gotKinds)
|
t.Fatalf("Kinds() = %#v, want [forecast]", gotKinds)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -69,7 +69,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
|
|||||||
if got[0].Schema != tt.wantSchema {
|
if got[0].Schema != tt.wantSchema {
|
||||||
t.Fatalf("Poll() schema = %q, want %q", got[0].Schema, tt.wantSchema)
|
t.Fatalf("Poll() schema = %q, want %q", got[0].Schema, tt.wantSchema)
|
||||||
}
|
}
|
||||||
if got[0].Kind != event.Kind("forecast") {
|
if got[0].Kind != event.Kind(standards.KindForecast) {
|
||||||
t.Fatalf("Poll() kind = %q, want forecast", got[0].Kind)
|
t.Fatalf("Poll() kind = %q, want forecast", got[0].Kind)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -117,7 +117,7 @@ func TestForecastSourcePollEffectiveAtFallbackOrder(t *testing.T) {
|
|||||||
}))
|
}))
|
||||||
defer srv.Close()
|
defer srv.Close()
|
||||||
|
|
||||||
src, err := NewHourlyForecastSource(forecastSourceConfig("nws_forecast_hourly", srv.URL))
|
src, err := NewHourlyForecastSource(forecastSourceConfig(DriverForecastHourly, srv.URL))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewHourlyForecastSource() error = %v", err)
|
t.Fatalf("NewHourlyForecastSource() error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -148,7 +148,7 @@ func TestForecastSourcePollMetadataDecodeFailureStillEmitsRawEvent(t *testing.T)
|
|||||||
}))
|
}))
|
||||||
defer srv.Close()
|
defer srv.Close()
|
||||||
|
|
||||||
src, err := NewNarrativeForecastSource(forecastSourceConfig("nws_forecast_narrative", srv.URL))
|
src, err := NewNarrativeForecastSource(forecastSourceConfig(DriverForecastNarrative, srv.URL))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewNarrativeForecastSource() error = %v", err)
|
t.Fatalf("NewNarrativeForecastSource() error = %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,9 +20,7 @@ type ObservationSource struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
|
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
|
||||||
const driver = "nws_observation"
|
hs, err := fksources.NewHTTPSource(DriverObservation, cfg, "application/geo+json, application/json")
|
||||||
|
|
||||||
hs, err := fksources.NewHTTPSource(driver, cfg, "application/geo+json, application/json")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -32,7 +30,9 @@ func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
|
|||||||
|
|
||||||
func (s *ObservationSource) Name() string { return s.http.Name }
|
func (s *ObservationSource) Name() string { return s.http.Name }
|
||||||
|
|
||||||
func (s *ObservationSource) Kinds() []event.Kind { return []event.Kind{event.Kind("observation")} }
|
func (s *ObservationSource) Kinds() []event.Kind {
|
||||||
|
return []event.Kind{event.Kind(standards.KindObservation)}
|
||||||
|
}
|
||||||
|
|
||||||
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
|
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||||
raw, meta, changed, err := s.fetchRaw(ctx)
|
raw, meta, changed, err := s.fetchRaw(ctx)
|
||||||
@@ -54,7 +54,7 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
|
|||||||
eventID := fksources.DefaultEventID(meta.ID, s.http.Name, effectiveAt, emittedAt)
|
eventID := fksources.DefaultEventID(meta.ID, s.http.Name, effectiveAt, emittedAt)
|
||||||
|
|
||||||
return fksources.SingleEvent(
|
return fksources.SingleEvent(
|
||||||
event.Kind("observation"),
|
event.Kind(standards.KindObservation),
|
||||||
s.http.Name,
|
s.http.Name,
|
||||||
standards.SchemaRawNWSObservationV1,
|
standards.SchemaRawNWSObservationV1,
|
||||||
eventID,
|
eventID,
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
|
|
||||||
"gitea.maximumdirect.net/ejr/feedkit/config"
|
"gitea.maximumdirect.net/ejr/feedkit/config"
|
||||||
"gitea.maximumdirect.net/ejr/feedkit/event"
|
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
|
func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
|
||||||
@@ -31,7 +32,7 @@ func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
|
|||||||
|
|
||||||
src, err := NewObservationSource(config.SourceConfig{
|
src, err := NewObservationSource(config.SourceConfig{
|
||||||
Name: "NWSObservationTest",
|
Name: "NWSObservationTest",
|
||||||
Driver: "nws_observation",
|
Driver: DriverObservation,
|
||||||
Mode: config.SourceModePoll,
|
Mode: config.SourceModePoll,
|
||||||
Params: map[string]any{
|
Params: map[string]any{
|
||||||
"url": srv.URL,
|
"url": srv.URL,
|
||||||
@@ -41,7 +42,7 @@ func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewObservationSource() error = %v", err)
|
t.Fatalf("NewObservationSource() error = %v", err)
|
||||||
}
|
}
|
||||||
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("observation") {
|
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindObservation) {
|
||||||
t.Fatalf("Kinds() = %#v, want [observation]", got)
|
t.Fatalf("Kinds() = %#v, want [observation]", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -52,7 +53,7 @@ func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
|
|||||||
if len(first) != 1 {
|
if len(first) != 1 {
|
||||||
t.Fatalf("first Poll() len = %d, want 1", len(first))
|
t.Fatalf("first Poll() len = %d, want 1", len(first))
|
||||||
}
|
}
|
||||||
if first[0].Kind != event.Kind("observation") {
|
if first[0].Kind != event.Kind(standards.KindObservation) {
|
||||||
t.Fatalf("first Poll() kind = %q", first[0].Kind)
|
t.Fatalf("first Poll() kind = %q", first[0].Kind)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -22,9 +22,7 @@ type WeatherStoriesSource struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewWeatherStoriesSource(cfg config.SourceConfig) (*WeatherStoriesSource, error) {
|
func NewWeatherStoriesSource(cfg config.SourceConfig) (*WeatherStoriesSource, error) {
|
||||||
const driver = "nws_weatherstories"
|
hs, err := fksources.NewHTTPSource(DriverWeatherStories, cfg, "application/geo+json, application/json")
|
||||||
|
|
||||||
hs, err := fksources.NewHTTPSource(driver, cfg, "application/geo+json, application/json")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -35,7 +33,7 @@ func NewWeatherStoriesSource(cfg config.SourceConfig) (*WeatherStoriesSource, er
|
|||||||
func (s *WeatherStoriesSource) Name() string { return s.http.Name }
|
func (s *WeatherStoriesSource) Name() string { return s.http.Name }
|
||||||
|
|
||||||
func (s *WeatherStoriesSource) Kinds() []event.Kind {
|
func (s *WeatherStoriesSource) Kinds() []event.Kind {
|
||||||
return []event.Kind{event.Kind("weather_story")}
|
return []event.Kind{event.Kind(standards.KindWeatherStory)}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *WeatherStoriesSource) Poll(ctx context.Context) ([]event.Event, error) {
|
func (s *WeatherStoriesSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||||
@@ -61,7 +59,7 @@ func (s *WeatherStoriesSource) Poll(ctx context.Context) ([]event.Event, error)
|
|||||||
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
||||||
|
|
||||||
return fksources.SingleEvent(
|
return fksources.SingleEvent(
|
||||||
event.Kind("weather_story"),
|
event.Kind(standards.KindWeatherStory),
|
||||||
s.http.Name,
|
s.http.Name,
|
||||||
standards.SchemaRawNWSWeatherStoriesV1,
|
standards.SchemaRawNWSWeatherStoriesV1,
|
||||||
eventID,
|
eventID,
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ func TestWeatherStoriesSourcePollEmitsExpectedEventAndPrefersLatestUpdateTime(t
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewWeatherStoriesSource() error = %v", err)
|
t.Fatalf("NewWeatherStoriesSource() error = %v", err)
|
||||||
}
|
}
|
||||||
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("weather_story") {
|
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindWeatherStory) {
|
||||||
t.Fatalf("Kinds() = %#v, want [weather_story]", got)
|
t.Fatalf("Kinds() = %#v, want [weather_story]", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -41,7 +41,7 @@ func TestWeatherStoriesSourcePollEmitsExpectedEventAndPrefersLatestUpdateTime(t
|
|||||||
}
|
}
|
||||||
|
|
||||||
got := events[0]
|
got := events[0]
|
||||||
if got.Kind != event.Kind("weather_story") {
|
if got.Kind != event.Kind(standards.KindWeatherStory) {
|
||||||
t.Fatalf("Kind = %q, want weather_story", got.Kind)
|
t.Fatalf("Kind = %q, want weather_story", got.Kind)
|
||||||
}
|
}
|
||||||
if got.Schema != standards.SchemaRawNWSWeatherStoriesV1 {
|
if got.Schema != standards.SchemaRawNWSWeatherStoriesV1 {
|
||||||
@@ -148,7 +148,7 @@ func TestWeatherStoriesSourcePollMetadataDecodeFailureStillEmitsRawEvent(t *test
|
|||||||
func weatherStoriesSourceConfig(url string) config.SourceConfig {
|
func weatherStoriesSourceConfig(url string) config.SourceConfig {
|
||||||
return config.SourceConfig{
|
return config.SourceConfig{
|
||||||
Name: "test-weatherstories-source",
|
Name: "test-weatherstories-source",
|
||||||
Driver: "nws_weatherstories",
|
Driver: DriverWeatherStories,
|
||||||
Mode: config.SourceModePoll,
|
Mode: config.SourceModePoll,
|
||||||
Params: map[string]any{
|
Params: map[string]any{
|
||||||
"url": url,
|
"url": url,
|
||||||
|
|||||||
7
internal/sources/openmeteo/driver.go
Normal file
7
internal/sources/openmeteo/driver.go
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
package openmeteo
|
||||||
|
|
||||||
|
// Source driver strings registered by weatherfeeder for Open-Meteo sources.
|
||||||
|
const (
|
||||||
|
DriverObservation = "openmeteo_observation"
|
||||||
|
DriverForecast = "openmeteo_forecast"
|
||||||
|
)
|
||||||
@@ -19,9 +19,7 @@ type ForecastSource struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewForecastSource(cfg config.SourceConfig) (*ForecastSource, error) {
|
func NewForecastSource(cfg config.SourceConfig) (*ForecastSource, error) {
|
||||||
const driver = "openmeteo_forecast"
|
hs, err := fksources.NewHTTPSource(DriverForecast, cfg, "application/json")
|
||||||
|
|
||||||
hs, err := fksources.NewHTTPSource(driver, cfg, "application/json")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -31,7 +29,9 @@ func NewForecastSource(cfg config.SourceConfig) (*ForecastSource, error) {
|
|||||||
|
|
||||||
func (s *ForecastSource) Name() string { return s.http.Name }
|
func (s *ForecastSource) Name() string { return s.http.Name }
|
||||||
|
|
||||||
func (s *ForecastSource) Kinds() []event.Kind { return []event.Kind{event.Kind("forecast")} }
|
func (s *ForecastSource) Kinds() []event.Kind {
|
||||||
|
return []event.Kind{event.Kind(standards.KindForecast)}
|
||||||
|
}
|
||||||
|
|
||||||
func (s *ForecastSource) Poll(ctx context.Context) ([]event.Event, error) {
|
func (s *ForecastSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||||
raw, meta, changed, err := s.fetchRaw(ctx)
|
raw, meta, changed, err := s.fetchRaw(ctx)
|
||||||
@@ -55,7 +55,7 @@ func (s *ForecastSource) Poll(ctx context.Context) ([]event.Event, error) {
|
|||||||
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
||||||
|
|
||||||
return fksources.SingleEvent(
|
return fksources.SingleEvent(
|
||||||
event.Kind("forecast"),
|
event.Kind(standards.KindForecast),
|
||||||
s.http.Name,
|
s.http.Name,
|
||||||
standards.SchemaRawOpenMeteoHourlyForecastV1,
|
standards.SchemaRawOpenMeteoHourlyForecastV1,
|
||||||
eventID,
|
eventID,
|
||||||
|
|||||||
@@ -19,9 +19,7 @@ type ObservationSource struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
|
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
|
||||||
const driver = "openmeteo_observation"
|
hs, err := fksources.NewHTTPSource(DriverObservation, cfg, "application/json")
|
||||||
|
|
||||||
hs, err := fksources.NewHTTPSource(driver, cfg, "application/json")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -31,7 +29,9 @@ func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
|
|||||||
|
|
||||||
func (s *ObservationSource) Name() string { return s.http.Name }
|
func (s *ObservationSource) Name() string { return s.http.Name }
|
||||||
|
|
||||||
func (s *ObservationSource) Kinds() []event.Kind { return []event.Kind{event.Kind("observation")} }
|
func (s *ObservationSource) Kinds() []event.Kind {
|
||||||
|
return []event.Kind{event.Kind(standards.KindObservation)}
|
||||||
|
}
|
||||||
|
|
||||||
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
|
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||||
raw, meta, changed, err := s.fetchRaw(ctx)
|
raw, meta, changed, err := s.fetchRaw(ctx)
|
||||||
@@ -52,7 +52,7 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
|
|||||||
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
||||||
|
|
||||||
return fksources.SingleEvent(
|
return fksources.SingleEvent(
|
||||||
event.Kind("observation"),
|
event.Kind(standards.KindObservation),
|
||||||
s.http.Name,
|
s.http.Name,
|
||||||
standards.SchemaRawOpenMeteoCurrentV1,
|
standards.SchemaRawOpenMeteoCurrentV1,
|
||||||
eventID,
|
eventID,
|
||||||
|
|||||||
@@ -5,12 +5,13 @@ import (
|
|||||||
|
|
||||||
"gitea.maximumdirect.net/ejr/feedkit/config"
|
"gitea.maximumdirect.net/ejr/feedkit/config"
|
||||||
"gitea.maximumdirect.net/ejr/feedkit/event"
|
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestObservationSourceAdvertisesKinds(t *testing.T) {
|
func TestObservationSourceAdvertisesKinds(t *testing.T) {
|
||||||
src, err := NewObservationSource(config.SourceConfig{
|
src, err := NewObservationSource(config.SourceConfig{
|
||||||
Name: "openmeteo-observation-test",
|
Name: "openmeteo-observation-test",
|
||||||
Driver: "openmeteo_observation",
|
Driver: DriverObservation,
|
||||||
Mode: config.SourceModePoll,
|
Mode: config.SourceModePoll,
|
||||||
Params: map[string]any{
|
Params: map[string]any{
|
||||||
"url": "https://example.invalid",
|
"url": "https://example.invalid",
|
||||||
@@ -20,7 +21,7 @@ func TestObservationSourceAdvertisesKinds(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewObservationSource() error = %v", err)
|
t.Fatalf("NewObservationSource() error = %v", err)
|
||||||
}
|
}
|
||||||
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("observation") {
|
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindObservation) {
|
||||||
t.Fatalf("Kinds() = %#v, want [observation]", got)
|
t.Fatalf("Kinds() = %#v, want [observation]", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -28,7 +29,7 @@ func TestObservationSourceAdvertisesKinds(t *testing.T) {
|
|||||||
func TestForecastSourceAdvertisesKinds(t *testing.T) {
|
func TestForecastSourceAdvertisesKinds(t *testing.T) {
|
||||||
src, err := NewForecastSource(config.SourceConfig{
|
src, err := NewForecastSource(config.SourceConfig{
|
||||||
Name: "openmeteo-forecast-test",
|
Name: "openmeteo-forecast-test",
|
||||||
Driver: "openmeteo_forecast",
|
Driver: DriverForecast,
|
||||||
Mode: config.SourceModePoll,
|
Mode: config.SourceModePoll,
|
||||||
Params: map[string]any{
|
Params: map[string]any{
|
||||||
"url": "https://example.invalid",
|
"url": "https://example.invalid",
|
||||||
@@ -38,7 +39,7 @@ func TestForecastSourceAdvertisesKinds(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewForecastSource() error = %v", err)
|
t.Fatalf("NewForecastSource() error = %v", err)
|
||||||
}
|
}
|
||||||
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("forecast") {
|
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindForecast) {
|
||||||
t.Fatalf("Kinds() = %#v, want [forecast]", got)
|
t.Fatalf("Kinds() = %#v, want [forecast]", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
6
internal/sources/openweather/driver.go
Normal file
6
internal/sources/openweather/driver.go
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
package openweather
|
||||||
|
|
||||||
|
// Source driver strings registered by weatherfeeder for OpenWeather sources.
|
||||||
|
const (
|
||||||
|
DriverObservation = "openweather_observation"
|
||||||
|
)
|
||||||
@@ -19,9 +19,7 @@ type ObservationSource struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
|
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
|
||||||
const driver = "openweather_observation"
|
hs, err := fksources.NewHTTPSource(DriverObservation, cfg, "application/json")
|
||||||
|
|
||||||
hs, err := fksources.NewHTTPSource(driver, cfg, "application/json")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -35,7 +33,9 @@ func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
|
|||||||
|
|
||||||
func (s *ObservationSource) Name() string { return s.http.Name }
|
func (s *ObservationSource) Name() string { return s.http.Name }
|
||||||
|
|
||||||
func (s *ObservationSource) Kinds() []event.Kind { return []event.Kind{event.Kind("observation")} }
|
func (s *ObservationSource) Kinds() []event.Kind {
|
||||||
|
return []event.Kind{event.Kind(standards.KindObservation)}
|
||||||
|
}
|
||||||
|
|
||||||
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
|
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||||
if err := owcommon.RequireMetricUnits(s.http.URL); err != nil {
|
if err := owcommon.RequireMetricUnits(s.http.URL); err != nil {
|
||||||
@@ -60,7 +60,7 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
|
|||||||
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
|
||||||
|
|
||||||
return fksources.SingleEvent(
|
return fksources.SingleEvent(
|
||||||
event.Kind("observation"),
|
event.Kind(standards.KindObservation),
|
||||||
s.http.Name,
|
s.http.Name,
|
||||||
standards.SchemaRawOpenWeatherCurrentV1,
|
standards.SchemaRawOpenWeatherCurrentV1,
|
||||||
eventID,
|
eventID,
|
||||||
|
|||||||
@@ -5,12 +5,13 @@ import (
|
|||||||
|
|
||||||
"gitea.maximumdirect.net/ejr/feedkit/config"
|
"gitea.maximumdirect.net/ejr/feedkit/config"
|
||||||
"gitea.maximumdirect.net/ejr/feedkit/event"
|
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestObservationSourceAdvertisesKinds(t *testing.T) {
|
func TestObservationSourceAdvertisesKinds(t *testing.T) {
|
||||||
src, err := NewObservationSource(config.SourceConfig{
|
src, err := NewObservationSource(config.SourceConfig{
|
||||||
Name: "openweather-observation-test",
|
Name: "openweather-observation-test",
|
||||||
Driver: "openweather_observation",
|
Driver: DriverObservation,
|
||||||
Mode: config.SourceModePoll,
|
Mode: config.SourceModePoll,
|
||||||
Params: map[string]any{
|
Params: map[string]any{
|
||||||
"url": "https://example.invalid?units=metric",
|
"url": "https://example.invalid?units=metric",
|
||||||
@@ -20,7 +21,7 @@ func TestObservationSourceAdvertisesKinds(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("NewObservationSource() error = %v", err)
|
t.Fatalf("NewObservationSource() error = %v", err)
|
||||||
}
|
}
|
||||||
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("observation") {
|
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindObservation) {
|
||||||
t.Fatalf("Kinds() = %#v, want [observation]", got)
|
t.Fatalf("Kinds() = %#v, want [observation]", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,12 +16,11 @@ import (
|
|||||||
fksources "gitea.maximumdirect.net/ejr/feedkit/sources"
|
fksources "gitea.maximumdirect.net/ejr/feedkit/sources"
|
||||||
"gitea.maximumdirect.net/ejr/feedkit/transport"
|
"gitea.maximumdirect.net/ejr/feedkit/transport"
|
||||||
spcprovider "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/spc"
|
spcprovider "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/spc"
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/internal/httpconfig"
|
||||||
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
driverConvectiveOutlook = "spc_convective_outlook"
|
|
||||||
|
|
||||||
acceptGeoJSON = "application/geo+json, application/json"
|
acceptGeoJSON = "application/geo+json, application/json"
|
||||||
acceptDiscussion = "text/html, application/xhtml+xml"
|
acceptDiscussion = "text/html, application/xhtml+xml"
|
||||||
acceptRSS = "application/rss+xml, application/xml, text/xml"
|
acceptRSS = "application/rss+xml, application/xml, text/xml"
|
||||||
@@ -56,53 +55,27 @@ type ConvectiveOutlookSource struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewConvectiveOutlookSource(cfg config.SourceConfig) (*ConvectiveOutlookSource, error) {
|
func NewConvectiveOutlookSource(cfg config.SourceConfig) (*ConvectiveOutlookSource, error) {
|
||||||
name := strings.TrimSpace(cfg.Name)
|
httpSettings, err := httpconfig.Parse(DriverConvectiveOutlook, cfg)
|
||||||
if name == "" {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%s: name is required", driverConvectiveOutlook)
|
return nil, err
|
||||||
}
|
|
||||||
if cfg.Params == nil {
|
|
||||||
return nil, fmt.Errorf("%s %q: params are required", driverConvectiveOutlook, name)
|
|
||||||
}
|
|
||||||
|
|
||||||
userAgent, ok := cfg.ParamString("user_agent", "userAgent")
|
|
||||||
if !ok {
|
|
||||||
return nil, fmt.Errorf("%s %q: params.user_agent is required", driverConvectiveOutlook, name)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
latitude, err := requireFloatParam(cfg, "latitude")
|
latitude, err := requireFloatParam(cfg, "latitude")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%s %q: %w", driverConvectiveOutlook, name, err)
|
return nil, fmt.Errorf("%s %q: %w", DriverConvectiveOutlook, httpSettings.Name, err)
|
||||||
}
|
}
|
||||||
longitude, err := requireFloatParam(cfg, "longitude")
|
longitude, err := requireFloatParam(cfg, "longitude")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%s %q: %w", driverConvectiveOutlook, name, err)
|
return nil, fmt.Errorf("%s %q: %w", DriverConvectiveOutlook, httpSettings.Name, err)
|
||||||
}
|
|
||||||
|
|
||||||
timeout := transport.DefaultHTTPTimeout
|
|
||||||
if _, exists := cfg.Params["http_timeout"]; exists {
|
|
||||||
var ok bool
|
|
||||||
timeout, ok = cfg.ParamDuration("http_timeout")
|
|
||||||
if !ok || timeout <= 0 {
|
|
||||||
return nil, fmt.Errorf("source %q: params.http_timeout must be a positive duration", name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
bodyLimit := transport.DefaultHTTPResponseBodyLimitBytes
|
|
||||||
if _, exists := cfg.Params["http_response_body_limit_bytes"]; exists {
|
|
||||||
rawLimit, ok := cfg.ParamInt("http_response_body_limit_bytes")
|
|
||||||
if !ok || rawLimit <= 0 {
|
|
||||||
return nil, fmt.Errorf("source %q: params.http_response_body_limit_bytes must be a positive integer", name)
|
|
||||||
}
|
|
||||||
bodyLimit = int64(rawLimit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
geoJSONProducts, err := configuredGeoJSONProducts(cfg)
|
geoJSONProducts, err := configuredGeoJSONProducts(cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%s %q: %w", driverConvectiveOutlook, name, err)
|
return nil, fmt.Errorf("%s %q: %w", DriverConvectiveOutlook, httpSettings.Name, err)
|
||||||
}
|
}
|
||||||
discussions, err := configuredDiscussionProducts(cfg)
|
discussions, err := configuredDiscussionProducts(cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%s %q: %w", driverConvectiveOutlook, name, err)
|
return nil, fmt.Errorf("%s %q: %w", DriverConvectiveOutlook, httpSettings.Name, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
rssURL := ""
|
rssURL := ""
|
||||||
@@ -114,14 +87,14 @@ func NewConvectiveOutlookSource(cfg config.SourceConfig) (*ConvectiveOutlookSour
|
|||||||
locationName, _ := cfg.ParamString("location_name", "locationName")
|
locationName, _ := cfg.ParamString("location_name", "locationName")
|
||||||
|
|
||||||
return &ConvectiveOutlookSource{
|
return &ConvectiveOutlookSource{
|
||||||
name: name,
|
name: httpSettings.Name,
|
||||||
userAgent: userAgent,
|
userAgent: httpSettings.UserAgent,
|
||||||
locationID: locationID,
|
locationID: locationID,
|
||||||
locationName: locationName,
|
locationName: locationName,
|
||||||
latitude: latitude,
|
latitude: latitude,
|
||||||
longitude: longitude,
|
longitude: longitude,
|
||||||
client: transport.NewHTTPClient(timeout),
|
client: transport.NewHTTPClient(httpSettings.Timeout),
|
||||||
bodyLimit: bodyLimit,
|
bodyLimit: httpSettings.BodyLimitBytes,
|
||||||
geoJSONProducts: geoJSONProducts,
|
geoJSONProducts: geoJSONProducts,
|
||||||
discussions: discussions,
|
discussions: discussions,
|
||||||
rssURL: rssURL,
|
rssURL: rssURL,
|
||||||
@@ -131,7 +104,7 @@ func NewConvectiveOutlookSource(cfg config.SourceConfig) (*ConvectiveOutlookSour
|
|||||||
func (s *ConvectiveOutlookSource) Name() string { return s.name }
|
func (s *ConvectiveOutlookSource) Name() string { return s.name }
|
||||||
|
|
||||||
func (s *ConvectiveOutlookSource) Kinds() []event.Kind {
|
func (s *ConvectiveOutlookSource) Kinds() []event.Kind {
|
||||||
return []event.Kind{event.Kind("outlook")}
|
return []event.Kind{event.Kind(standards.KindOutlook)}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *ConvectiveOutlookSource) Poll(ctx context.Context) ([]event.Event, error) {
|
func (s *ConvectiveOutlookSource) Poll(ctx context.Context) ([]event.Event, error) {
|
||||||
@@ -221,7 +194,7 @@ func (s *ConvectiveOutlookSource) Poll(ctx context.Context) ([]event.Event, erro
|
|||||||
eventID := fksources.DefaultEventID("", s.name, &effectiveAt, emittedAt)
|
eventID := fksources.DefaultEventID("", s.name, &effectiveAt, emittedAt)
|
||||||
|
|
||||||
return fksources.SingleEvent(
|
return fksources.SingleEvent(
|
||||||
event.Kind("outlook"),
|
event.Kind(standards.KindOutlook),
|
||||||
s.name,
|
s.name,
|
||||||
standards.SchemaRawSPCConvectiveOutlookV1,
|
standards.SchemaRawSPCConvectiveOutlookV1,
|
||||||
eventID,
|
eventID,
|
||||||
|
|||||||
@@ -4,8 +4,6 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -22,7 +20,7 @@ func TestConvectiveOutlookSourceKinds(t *testing.T) {
|
|||||||
t.Fatalf("NewConvectiveOutlookSource() error = %v", err)
|
t.Fatalf("NewConvectiveOutlookSource() error = %v", err)
|
||||||
}
|
}
|
||||||
got := src.Kinds()
|
got := src.Kinds()
|
||||||
if len(got) != 1 || got[0] != event.Kind("outlook") {
|
if len(got) != 1 || got[0] != event.Kind(standards.KindOutlook) {
|
||||||
t.Fatalf("Kinds() = %#v, want [outlook]", got)
|
t.Fatalf("Kinds() = %#v, want [outlook]", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -57,7 +55,7 @@ func TestConvectiveOutlookSourcePollEmitsRawBundle(t *testing.T) {
|
|||||||
t.Fatalf("Poll() returned %d events, want 1", len(events))
|
t.Fatalf("Poll() returned %d events, want 1", len(events))
|
||||||
}
|
}
|
||||||
got := events[0]
|
got := events[0]
|
||||||
if got.Kind != event.Kind("outlook") {
|
if got.Kind != event.Kind(standards.KindOutlook) {
|
||||||
t.Fatalf("Kind = %q, want outlook", got.Kind)
|
t.Fatalf("Kind = %q, want outlook", got.Kind)
|
||||||
}
|
}
|
||||||
if got.Schema != standards.SchemaRawSPCConvectiveOutlookV1 {
|
if got.Schema != standards.SchemaRawSPCConvectiveOutlookV1 {
|
||||||
@@ -78,8 +76,8 @@ func TestConvectiveOutlookSourcePollEmitsRawBundle(t *testing.T) {
|
|||||||
if bundle.Latitude != 38.6239 || bundle.Longitude != -90.3571 {
|
if bundle.Latitude != 38.6239 || bundle.Longitude != -90.3571 {
|
||||||
t.Fatalf("coordinates = %v,%v", bundle.Latitude, bundle.Longitude)
|
t.Fatalf("coordinates = %v,%v", bundle.Latitude, bundle.Longitude)
|
||||||
}
|
}
|
||||||
if len(bundle.Products) != 12 {
|
if len(bundle.Products) != 9 {
|
||||||
t.Fatalf("Products length = %d, want 12", len(bundle.Products))
|
t.Fatalf("Products length = %d, want 9", len(bundle.Products))
|
||||||
}
|
}
|
||||||
if len(bundle.Discussions) != 3 {
|
if len(bundle.Discussions) != 3 {
|
||||||
t.Fatalf("Discussions length = %d, want 3", len(bundle.Discussions))
|
t.Fatalf("Discussions length = %d, want 3", len(bundle.Discussions))
|
||||||
@@ -277,7 +275,7 @@ func convectiveOutlookConfig(extra map[string]any) config.SourceConfig {
|
|||||||
}
|
}
|
||||||
return config.SourceConfig{
|
return config.SourceConfig{
|
||||||
Name: "spc-test",
|
Name: "spc-test",
|
||||||
Driver: driverConvectiveOutlook,
|
Driver: DriverConvectiveOutlook,
|
||||||
Mode: config.SourceModePoll,
|
Mode: config.SourceModePoll,
|
||||||
Params: params,
|
Params: params,
|
||||||
}
|
}
|
||||||
@@ -312,7 +310,7 @@ func geoJSONFixtureForProduct(t *testing.T, key string, blankIssueISO bool) []by
|
|||||||
case strings.HasPrefix(key, "day2_"):
|
case strings.HasPrefix(key, "day2_"):
|
||||||
name = "day2_torn.geojson"
|
name = "day2_torn.geojson"
|
||||||
case strings.HasPrefix(key, "day3_"):
|
case strings.HasPrefix(key, "day3_"):
|
||||||
name = "day3_wind.geojson"
|
name = "day3_cat.geojson"
|
||||||
default:
|
default:
|
||||||
t.Fatalf("unknown product key %q", key)
|
t.Fatalf("unknown product key %q", key)
|
||||||
}
|
}
|
||||||
@@ -340,16 +338,6 @@ func discussionFixtureForProduct(t *testing.T, key string) []byte {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func readSPCTestFixture(t *testing.T, name string) []byte {
|
|
||||||
t.Helper()
|
|
||||||
path := filepath.Join("..", "..", "providers", "spc", "testdata", name)
|
|
||||||
raw, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read fixture %s: %v", path, err)
|
|
||||||
}
|
|
||||||
return raw
|
|
||||||
}
|
|
||||||
|
|
||||||
const testRSS = `<?xml version="1.0"?>
|
const testRSS = `<?xml version="1.0"?>
|
||||||
<rss version="2.0">
|
<rss version="2.0">
|
||||||
<channel>
|
<channel>
|
||||||
|
|||||||
6
internal/sources/spc/driver.go
Normal file
6
internal/sources/spc/driver.go
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
package spc
|
||||||
|
|
||||||
|
// Source driver strings registered by weatherfeeder for SPC sources.
|
||||||
|
const (
|
||||||
|
DriverConvectiveOutlook = "spc_convective_outlook"
|
||||||
|
)
|
||||||
17
internal/sources/spc/fixture_test.go
Normal file
17
internal/sources/spc/fixture_test.go
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
package spc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func readSPCTestFixture(t *testing.T, name string) []byte {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join("..", "..", "providers", "spc", "testdata", name)
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read fixture %s: %v", path, err)
|
||||||
|
}
|
||||||
|
return raw
|
||||||
|
}
|
||||||
38
model/docs_test.go
Normal file
38
model/docs_test.go
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
package model
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDocumentedConsumerModelTypes(t *testing.T) {
|
||||||
|
raw, err := os.ReadFile("../docs/consumers/pkg-model.md")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile(pkg-model.md) error = %v", err)
|
||||||
|
}
|
||||||
|
doc := string(raw)
|
||||||
|
|
||||||
|
required := []string{
|
||||||
|
"gitea.maximumdirect.net/ejr/weatherfeeder/model",
|
||||||
|
"WeatherObservation",
|
||||||
|
"WeatherForecastRun",
|
||||||
|
"WeatherForecastPeriod",
|
||||||
|
"WeatherForecastDiscussion",
|
||||||
|
"WeatherForecastDiscussionSection",
|
||||||
|
"WeatherStoryRun",
|
||||||
|
"WeatherStory",
|
||||||
|
"WeatherAlertRun",
|
||||||
|
"WeatherAlert",
|
||||||
|
"WeatherAlertReference",
|
||||||
|
"WeatherOutlookRun",
|
||||||
|
"WeatherOutlookDiscussion",
|
||||||
|
"WeatherOutlook",
|
||||||
|
"WMOCode",
|
||||||
|
}
|
||||||
|
for _, want := range required {
|
||||||
|
if !strings.Contains(doc, want) {
|
||||||
|
t.Fatalf("docs/consumers/pkg-model.md missing %q", want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -8,13 +8,23 @@ import (
|
|||||||
// WeatherOutlookRun is a snapshot of convective outlook polygons for a
|
// WeatherOutlookRun is a snapshot of convective outlook polygons for a
|
||||||
// configured location as-of a provider issue time.
|
// configured location as-of a provider issue time.
|
||||||
type WeatherOutlookRun struct {
|
type WeatherOutlookRun struct {
|
||||||
LocationID string `json:"locationId,omitempty"`
|
LocationID string `json:"locationId,omitempty"`
|
||||||
LocationName string `json:"locationName,omitempty"`
|
LocationName string `json:"locationName,omitempty"`
|
||||||
Latitude *float64 `json:"latitude,omitempty"`
|
Latitude *float64 `json:"latitude,omitempty"`
|
||||||
Longitude *float64 `json:"longitude,omitempty"`
|
Longitude *float64 `json:"longitude,omitempty"`
|
||||||
AsOf time.Time `json:"asOf"`
|
AsOf time.Time `json:"asOf"`
|
||||||
IssuedAt *time.Time `json:"issuedAt,omitempty"`
|
IssuedAt *time.Time `json:"issuedAt,omitempty"`
|
||||||
Outlooks []WeatherOutlook `json:"outlooks"`
|
Outlooks []WeatherOutlook `json:"outlooks"`
|
||||||
|
Discussions []WeatherOutlookDiscussion `json:"discussions"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// WeatherOutlookDiscussion is run-level SPC outlook prose for one outlook day.
|
||||||
|
type WeatherOutlookDiscussion struct {
|
||||||
|
Day int `json:"day"`
|
||||||
|
Headline string `json:"headline,omitempty"`
|
||||||
|
Summary string `json:"summary,omitempty"`
|
||||||
|
Discussion string `json:"discussion,omitempty"`
|
||||||
|
UpdatedAt *time.Time `json:"updatedAt,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// WeatherOutlook is a canonical representation of one outlook polygon.
|
// WeatherOutlook is a canonical representation of one outlook polygon.
|
||||||
@@ -32,9 +42,6 @@ type WeatherOutlook struct {
|
|||||||
IssuedAt time.Time `json:"issuedAt"`
|
IssuedAt time.Time `json:"issuedAt"`
|
||||||
ExpiresAt time.Time `json:"expiresAt"`
|
ExpiresAt time.Time `json:"expiresAt"`
|
||||||
Forecaster string `json:"forecaster,omitempty"`
|
Forecaster string `json:"forecaster,omitempty"`
|
||||||
Headline string `json:"headline,omitempty"`
|
|
||||||
Summary string `json:"summary,omitempty"`
|
|
||||||
Discussion string `json:"discussion,omitempty"`
|
|
||||||
SourceURL string `json:"sourceUrl,omitempty"`
|
SourceURL string `json:"sourceUrl,omitempty"`
|
||||||
ImageURL string `json:"imageUrl,omitempty"`
|
ImageURL string `json:"imageUrl,omitempty"`
|
||||||
ContainsLocation bool `json:"containsLocation"`
|
ContainsLocation bool `json:"containsLocation"`
|
||||||
|
|||||||
60
model/outlook_test.go
Normal file
60
model/outlook_test.go
Normal file
@@ -0,0 +1,60 @@
|
|||||||
|
package model
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestWeatherOutlookJSONShape(t *testing.T) {
|
||||||
|
updatedAt := time.Date(2026, 6, 11, 16, 30, 0, 0, time.UTC)
|
||||||
|
run := WeatherOutlookRun{
|
||||||
|
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
|
||||||
|
Outlooks: []WeatherOutlook{{
|
||||||
|
ID: "outlook-1",
|
||||||
|
Provider: "spc",
|
||||||
|
Product: "convective",
|
||||||
|
Day: 1,
|
||||||
|
OutlookType: "categorical",
|
||||||
|
Label: "SLGT",
|
||||||
|
ValidFrom: time.Date(2026, 6, 11, 13, 0, 0, 0, time.UTC),
|
||||||
|
ValidTo: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||||
|
IssuedAt: time.Date(2026, 6, 11, 12, 34, 56, 0, time.UTC),
|
||||||
|
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
|
||||||
|
ContainsLocation: true,
|
||||||
|
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-91,38],[-90,38],[-90,39],[-91,39],[-91,38]]]}`),
|
||||||
|
}},
|
||||||
|
Discussions: []WeatherOutlookDiscussion{{
|
||||||
|
Day: 1,
|
||||||
|
Headline: "Day 1 Convective Outlook",
|
||||||
|
Summary: "Severe thunderstorms are possible.",
|
||||||
|
Discussion: "Full discussion text.",
|
||||||
|
UpdatedAt: &updatedAt,
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
|
||||||
|
raw, err := json.Marshal(run)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Marshal(WeatherOutlookRun) error = %v", err)
|
||||||
|
}
|
||||||
|
got := string(raw)
|
||||||
|
|
||||||
|
for _, want := range []string{`"outlooks"`, `"discussions"`, `"headline"`, `"summary"`, `"discussion"`, `"updatedAt"`} {
|
||||||
|
if !strings.Contains(got, want) {
|
||||||
|
t.Fatalf("WeatherOutlookRun JSON missing %s: %s", want, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
outlookStart := strings.Index(got, `"outlooks"`)
|
||||||
|
discussionStart := strings.Index(got, `"discussions"`)
|
||||||
|
if outlookStart == -1 || discussionStart == -1 || discussionStart <= outlookStart {
|
||||||
|
t.Fatalf("WeatherOutlookRun JSON has unexpected outlook/discussion order: %s", got)
|
||||||
|
}
|
||||||
|
outlookJSON := got[outlookStart:discussionStart]
|
||||||
|
for _, unwanted := range []string{`"headline"`, `"summary"`, `"discussion"`} {
|
||||||
|
if strings.Contains(outlookJSON, unwanted) {
|
||||||
|
t.Fatalf("WeatherOutlook JSON contains polygon-level prose key %s: %s", unwanted, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
106
standards/docs_test.go
Normal file
106
standards/docs_test.go
Normal file
@@ -0,0 +1,106 @@
|
|||||||
|
package standards
|
||||||
|
|
||||||
|
import (
|
||||||
|
"go/ast"
|
||||||
|
"go/parser"
|
||||||
|
"go/token"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDocumentedEventSchemas(t *testing.T) {
|
||||||
|
raw, err := os.ReadFile("../docs/integrations/events.md")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile(events.md) error = %v", err)
|
||||||
|
}
|
||||||
|
doc := string(raw)
|
||||||
|
|
||||||
|
for _, schema := range schemaConstants(t, false) {
|
||||||
|
if !strings.Contains(doc, schema) {
|
||||||
|
t.Fatalf("docs/integrations/events.md missing schema %q", schema)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDocumentedConsumerStandardsConstants(t *testing.T) {
|
||||||
|
raw, err := os.ReadFile("../docs/consumers/pkg-standards.md")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadFile(pkg-standards.md) error = %v", err)
|
||||||
|
}
|
||||||
|
doc := string(raw)
|
||||||
|
|
||||||
|
for _, schema := range schemaConstants(t, true) {
|
||||||
|
if !strings.Contains(doc, schema) {
|
||||||
|
t.Fatalf("docs/consumers/pkg-standards.md missing schema %q", schema)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, kind := range kindConstants(t) {
|
||||||
|
if !strings.Contains(doc, kind) {
|
||||||
|
t.Fatalf("docs/consumers/pkg-standards.md missing kind %q", kind)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func schemaConstants(t *testing.T, includeNonCurrent bool) []string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
return stringConstantsFromFile(t, "schema.go", "Schema", func(name string) bool {
|
||||||
|
return !includeNonCurrent && schemaConstantNotInCurrentContract(name)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func kindConstants(t *testing.T) []string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
return stringConstantsFromFile(t, "kind.go", "Kind", nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func stringConstantsFromFile(t *testing.T, path string, prefix string, skip func(string) bool) []string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
file, err := parser.ParseFile(token.NewFileSet(), path, nil, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseFile(%s) error = %v", path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var out []string
|
||||||
|
ast.Inspect(file, func(n ast.Node) bool {
|
||||||
|
valueSpec, ok := n.(*ast.ValueSpec)
|
||||||
|
if !ok {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for i, name := range valueSpec.Names {
|
||||||
|
if !strings.HasPrefix(name.Name, prefix) || (skip != nil && skip(name.Name)) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if i >= len(valueSpec.Values) {
|
||||||
|
t.Fatalf("constant %s has no explicit value", name.Name)
|
||||||
|
}
|
||||||
|
lit, ok := valueSpec.Values[i].(*ast.BasicLit)
|
||||||
|
if !ok || lit.Kind != token.STRING {
|
||||||
|
t.Fatalf("constant %s is not a string literal", name.Name)
|
||||||
|
}
|
||||||
|
value, err := strconv.Unquote(lit.Value)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("constant %s value is not a quoted string: %v", name.Name, err)
|
||||||
|
}
|
||||||
|
out = append(out, value)
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
if len(out) == 0 {
|
||||||
|
t.Fatalf("no %s constants found in %s", prefix, path)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func schemaConstantNotInCurrentContract(name string) bool {
|
||||||
|
switch name {
|
||||||
|
case "SchemaRawOpenWeatherHourlyForecastV1":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
11
standards/kind.go
Normal file
11
standards/kind.go
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
package standards
|
||||||
|
|
||||||
|
// Event kind strings used by weatherfeeder events and routing policy.
|
||||||
|
const (
|
||||||
|
KindObservation = "observation"
|
||||||
|
KindForecast = "forecast"
|
||||||
|
KindForecastDiscussion = "forecast_discussion"
|
||||||
|
KindWeatherStory = "weather_story"
|
||||||
|
KindAlert = "alert"
|
||||||
|
KindOutlook = "outlook"
|
||||||
|
)
|
||||||
@@ -33,4 +33,5 @@ const (
|
|||||||
SchemaWeatherStoryV1 = "weather.weather_story.v1"
|
SchemaWeatherStoryV1 = "weather.weather_story.v1"
|
||||||
SchemaWeatherAlertV1 = "weather.alert.v1"
|
SchemaWeatherAlertV1 = "weather.alert.v1"
|
||||||
SchemaWeatherOutlookV1 = "weather.outlook.v1"
|
SchemaWeatherOutlookV1 = "weather.outlook.v1"
|
||||||
|
SchemaWeatherOutlookV2 = "weather.outlook.v2"
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user