diff --git a/docs/consumers/api.md b/docs/consumers/api.md index 3813a5e..c635f80 100644 --- a/docs/consumers/api.md +++ b/docs/consumers/api.md @@ -63,7 +63,7 @@ func Decode(payload []byte) (any, error) { case standards.SchemaWeatherAlertV1: var out model.WeatherAlertRun return &out, json.Unmarshal(evt.Payload, &out) - case standards.SchemaWeatherOutlookV1: + case standards.SchemaWeatherOutlookV2: var out model.WeatherOutlookRun return &out, json.Unmarshal(evt.Payload, &out) default: diff --git a/docs/consumers/pkg-model.md b/docs/consumers/pkg-model.md index c07a6ea..cca1a89 100644 --- a/docs/consumers/pkg-model.md +++ b/docs/consumers/pkg-model.md @@ -9,8 +9,8 @@ import "gitea.maximumdirect.net/ejr/weatherfeeder/model" ## Purpose Package `model` defines `weatherfeeder`'s canonical weather payload structs. -These structs are emitted as the `payload` of canonical `weather.*.v1` events -and are also the domain types consumed by downstream applications such as +These structs are emitted as the `payload` of canonical `weather.*` events and +are also the domain types consumed by downstream applications such as `weatherapi`. The JSON field tags on these structs are part of the wire contract. For the full @@ -27,7 +27,7 @@ Current canonical schema families map to these public types: | `weather.forecast_discussion.v1` | `WeatherForecastDiscussion` | | `weather.weather_story.v1` | `WeatherStoryRun` | | `weather.alert.v1` | `WeatherAlertRun` | -| `weather.outlook.v1` | `WeatherOutlookRun` | +| `weather.outlook.v2` | `WeatherOutlookRun` | Related child types include: @@ -41,6 +41,9 @@ Related child types include: - `WeatherOutlookDiscussion` - `WMOCode` +`WeatherOutlookRun` includes `WeatherOutlookDiscussion` entries as run-level +SPC day discussions. + ## Wire And Compatibility Rules - JSON tags define canonical payload field names. diff --git a/docs/consumers/pkg-standards.md b/docs/consumers/pkg-standards.md index 2c19df7..a0f576e 100644 --- a/docs/consumers/pkg-standards.md +++ b/docs/consumers/pkg-standards.md @@ -41,13 +41,13 @@ Canonical schemas emitted after normalization: | `SchemaWeatherForecastDiscussionV1` | `weather.forecast_discussion.v1` | | `SchemaWeatherStoryV1` | `weather.weather_story.v1` | | `SchemaWeatherAlertV1` | `weather.alert.v1` | -| `SchemaWeatherOutlookV1` | `weather.outlook.v1` | +| `SchemaWeatherOutlookV2` | `weather.outlook.v2` | -Additional canonical schema constant: +Historical canonical schema constant: | Constant | Value | |---|---| -| `SchemaWeatherOutlookV2` | `weather.outlook.v2` | +| `SchemaWeatherOutlookV1` | `weather.outlook.v1` | ## Raw Schema Constants diff --git a/docs/integrations/events.md b/docs/integrations/events.md index 5d2dc79..9b9c9ff 100644 --- a/docs/integrations/events.md +++ b/docs/integrations/events.md @@ -37,7 +37,7 @@ Canonical schemas emitted after normalization: | `forecast_discussion` | `weather.forecast_discussion.v1` | `WeatherForecastDiscussion` | | `weather_story` | `weather.weather_story.v1` | `WeatherStoryRun` | | `alert` | `weather.alert.v1` | `WeatherAlertRun` | -| `outlook` | `weather.outlook.v1` | `WeatherOutlookRun` | +| `outlook` | `weather.outlook.v2` | `WeatherOutlookRun` | Raw upstream schemas emitted by current sources: @@ -220,13 +220,15 @@ Payload type: `WeatherAlertRun`. `references[]` entries contain optional `id`, `identifier`, `sender`, and `sent` fields. -## `weather.outlook.v1` +## `weather.outlook.v2` Payload type: `WeatherOutlookRun`. The current producer is the SPC convective outlook normalizer. It emits Day 1-3 -convective outlook polygons for categorical, tornado, hail, and wind products. -All timestamps are UTC. +convective outlook snapshots for categorical, tornado, hail, and wind products +that apply to the configured forecast point. Raw SPC bundles remain complete; +canonical outlook payloads are filtered to local polygons. All timestamps are +UTC. | Field | Type | Required | Notes | |---|---|:---:|---| @@ -236,7 +238,8 @@ All timestamps are UTC. | `longitude` | number | no | Configured point longitude in decimal degrees. | | `asOf` | timestamp | yes | Snapshot time. For SPC, this is the latest outlook issue time when available. | | `issuedAt` | timestamp | no | Latest issue time across outlook features when any feature exists. | -| `outlooks` | array | yes | Ordered outlook polygons. | +| `outlooks` | array | yes | Ordered outlook polygons containing the configured point. | +| `discussions` | array | yes | Run-level day discussions for retained outlook days. | `outlooks[]` entries: @@ -255,19 +258,53 @@ All timestamps are UTC. | `issuedAt` | timestamp | yes | Feature issue time. | | `expiresAt` | timestamp | yes | Expiration time; currently equal to `validTo`. | | `forecaster` | string | no | SPC forecaster text, when present. | -| `headline` | string | no | Matching Day 1-3 print-page product title. | -| `summary` | string | no | Text from the print-page `...SUMMARY...` section. | -| `discussion` | string | no | Cleaned full print-page product text. | | `sourceUrl` | string | no | GeoJSON product URL for this outlook feature. | | `imageUrl` | string | no | Reserved for provider image URLs; currently empty. | -| `containsLocation` | boolean | yes | Whether the configured point is inside or on the boundary of the polygon. | +| `containsLocation` | boolean | yes | Always `true` for emitted v2 outlooks. | | `geometry` | object | yes | Compact GeoJSON `Polygon` or `MultiPolygon` geometry. | `geometry` preserves the SPC feature geometry as compact GeoJSON using `[longitude, latitude]` coordinate order. `containsLocation` is computed with that geometry and the configured source `latitude`/`longitude`; boundary points -count as contained. All outlook polygons are emitted, including polygons that do -not contain the configured point. +count as contained. Polygons that do not contain the configured point are not +included in canonical v2 payloads. + +`discussions[]` entries: + +| Field | Type | Required | Notes | +|---|---|:---:|---| +| `day` | integer | yes | SPC outlook day, currently `1`, `2`, or `3`. | +| `headline` | string | no | Matching Day 1-3 print-page product title. | +| `summary` | string | no | Text from the print-page `...SUMMARY...` section. | +| `discussion` | string | no | Cleaned full print-page product text. | +| `updatedAt` | timestamp | no | Print-page update time, when present. | + +When no SPC polygons apply locally, the run is still emitted with `outlooks: []` +and `discussions: []`. Discussions are included only for days represented by at +least one retained outlook, and multiple retained outlook types for the same day +share one discussion entry. + +### SPC Outlook Supersession + +Consumers should prefer latest-run semantics for current conditions: read the +latest `WeatherOutlookRun` for the configured location and use its `outlooks` +and `discussions` arrays together. + +Historical SQL consumers that collapse older rows should identify superseded +outlooks by `provider`, `product`, `outlookType`, `validFrom`, and `validTo`, +then keep rows with the greatest `issuedAt`. `day` and `label` are not identity +fields. When multiple retained polygons share that latest `issuedAt`, preserve +the full group. + +## Legacy `weather.outlook.v1` + +`weather.outlook.v1` is a historical canonical schema retained as a standards +constant for older data and consumers. Current SPC normalization emits +`weather.outlook.v2`. + +The v1 payload used `WeatherOutlookRun` and placed `headline`, `summary`, and +`discussion` on each `outlooks[]` polygon. It also represented the pre-v2 SPC +canonical behavior, where national polygons were preserved in canonical output. ## Compact Example diff --git a/docs/integrations/postgres.md b/docs/integrations/postgres.md index 0a1d894..6bf429a 100644 --- a/docs/integrations/postgres.md +++ b/docs/integrations/postgres.md @@ -23,7 +23,7 @@ Events are mapped only for canonical weather schemas: - `weather.forecast_discussion.v1` - `weather.weather_story.v1` - `weather.alert.v1` -- `weather.outlook.v1` +- `weather.outlook.v2` Unsupported schemas produce no writes for this sink. Mapped events are inserted transactionally. Inserts use ordinary `INSERT`; duplicate primary keys fail the @@ -59,6 +59,7 @@ Parent tables store the feed event envelope: | `alert_references` | `run_event_id`, `alert_index`, `reference_index` | `as_of` | | `outlook_runs` | `event_id` | `as_of` | | `outlooks` | `run_event_id`, `outlook_index` | `as_of` | +| `outlook_discussions` | `run_event_id`, `discussion_index` | `as_of` | ## Table Contract @@ -408,6 +409,7 @@ Indexes: | `as_of` | `TIMESTAMPTZ` | no | `payload.asOf` | | `issued_at` | `TIMESTAMPTZ` | yes | `payload.issuedAt` | | `outlook_count` | `INTEGER` | no | `len(payload.outlooks)` | +| `discussion_count` | `INTEGER` | no | `len(payload.discussions)` | ### `outlooks` @@ -442,14 +444,36 @@ Indexes: | `issued_at` | `TIMESTAMPTZ` | no | `payload.outlooks[].issuedAt` | | `expires_at` | `TIMESTAMPTZ` | no | `payload.outlooks[].expiresAt` | | `forecaster` | `TEXT` | yes | `payload.outlooks[].forecaster` | -| `headline` | `TEXT` | yes | `payload.outlooks[].headline` | -| `summary` | `TEXT` | yes | `payload.outlooks[].summary` | -| `discussion` | `TEXT` | yes | `payload.outlooks[].discussion` | | `source_url` | `TEXT` | yes | `payload.outlooks[].sourceUrl` | | `image_url` | `TEXT` | yes | `payload.outlooks[].imageUrl` | | `contains_location` | `BOOLEAN` | no | `payload.outlooks[].containsLocation` | | `geometry_json` | `TEXT` | no | Compact JSON from `payload.outlooks[].geometry` | +### `outlook_discussions` + +Primary key: `run_event_id`, `discussion_index` + +Prune column: `as_of` + +Foreign key: `run_event_id` references `outlook_runs(event_id)` with cascade +delete. + +Indexes: + +- `idx_wf_outlook_discussions_day_as_of` on `day`, `as_of` +- unique `idx_wf_outlook_discussions_run_day` on `run_event_id`, `day` + +| Column | Type | Null | Source | +|---|---|:---:|---| +| `run_event_id` | `TEXT REFERENCES outlook_runs(event_id) ON DELETE CASCADE` | no | Parent event ID. | +| `discussion_index` | `INTEGER` | no | `payload.discussions[]` index. | +| `as_of` | `TIMESTAMPTZ` | no | Parent `payload.asOf` | +| `day` | `INTEGER` | no | `payload.discussions[].day` | +| `headline` | `TEXT` | yes | `payload.discussions[].headline` | +| `summary` | `TEXT` | yes | `payload.discussions[].summary` | +| `discussion` | `TEXT` | yes | `payload.discussions[].discussion` | +| `updated_at` | `TIMESTAMPTZ` | yes | `payload.discussions[].updatedAt` | + ## Retention When sink param `prune` is set, every successful write transaction deletes rows @@ -472,5 +496,6 @@ binary does not provide CLI commands for them. - `WeatherAlertRun`: read `alert_runs`, join `alerts` by `run_event_id` ordered by `alert_index`, then join `alert_references` by `run_event_id` and `alert_index` ordered by `reference_index`. -- `WeatherOutlookRun`: read `outlook_runs`, then join `outlooks` by - `run_event_id` ordered by `outlook_index`. +- `WeatherOutlookRun`: read `outlook_runs`, join `outlooks` by `run_event_id` + ordered by `outlook_index`, then join `outlook_discussions` by + `run_event_id` ordered by `discussion_index`. diff --git a/docs/integrations/spc.md b/docs/integrations/spc.md index 094a306..e9b6d6f 100644 --- a/docs/integrations/spc.md +++ b/docs/integrations/spc.md @@ -13,7 +13,7 @@ events are documented in [event wire contract](events.md). | Driver | Kind | Raw schema | Canonical schema | | --- | --- | --- | --- | -| `spc_convective_outlook` | `outlook` | `raw.spc.convective_outlook.v1` | `weather.outlook.v1` | +| `spc_convective_outlook` | `outlook` | `raw.spc.convective_outlook.v1` | `weather.outlook.v2` | ## Config Requirements @@ -52,8 +52,8 @@ It also fetches three required print pages: - Day 3 convective outlook print page GeoJSON products are authoritative for outlook polygons, valid windows, issue -times, labels, and severity rank. Print pages are authoritative for headline, -summary, and discussion text. +times, labels, and severity rank. Print pages are authoritative for run-level +day discussion headline, summary, and discussion text. ## Accept Headers @@ -99,23 +99,33 @@ Raw source `effective_at` prefers: 4. fetch time. The normalizer sets canonical `asOf` and normalized event `effective_at` from -the latest valid outlook feature `issuedAt`, with fallback to print-page update -time and then the incoming event envelope. +the latest valid GeoJSON issue time across the complete raw bundle, including +empty no-risk placeholders, with fallback to print-page update time and then the +incoming event envelope. ## Mapping Notes -Each GeoJSON feature becomes one canonical outlook. Products are ordered by day, -then categorical, tornado, hail, and wind. Feature order is preserved within -each product. +The raw source fetches and envelopes the complete SPC bundle. The normalizer +decodes every configured GeoJSON product, skips empty no-risk +`GeometryCollection` placeholders, and emits canonical outlooks only when the +configured point is inside or on the boundary of a real feature. Products are +ordered by day, then categorical, tornado, hail, and wind. Retained feature order +is preserved within each product. The normalizer computes `containsLocation` with the configured latitude and longitude against compact GeoJSON `Polygon` or `MultiPolygon` geometry. Coordinates use GeoJSON order, `[longitude, latitude]`, and boundary points count as contained. -All outlook polygons are preserved, including polygons that do not contain the -configured point. Matching print-page headline, summary, and discussion text is -attached to every outlook for the same day. +Canonical outlook runs are emitted even when no polygons apply locally. In that +case the payload contains empty `outlooks` and `discussions` arrays. + +Print-page prose is represented as run-level day discussions. Discussions are +included only for days represented by at least one retained outlook. Multiple +retained outlook types for the same day share one discussion entry. + +For downstream current-state and historical supersession guidance, see the +[event wire contract](events.md#spc-outlook-supersession). ## Failure Behavior diff --git a/docs/internal/postgres-sink.md b/docs/internal/postgres-sink.md index f15612c..46fc327 100644 --- a/docs/internal/postgres-sink.md +++ b/docs/internal/postgres-sink.md @@ -19,11 +19,16 @@ Inputs are canonical feed events. The mapper currently handles these schemas: - `weather.forecast_discussion.v1` - `weather.weather_story.v1` - `weather.alert.v1` -- `weather.outlook.v1` +- `weather.outlook.v2` Outputs are feedkit `PostgresWrite` values for weatherfeeder-owned tables. Unsupported schemas produce no writes and no error. +Outlook runs are written to `outlook_runs`, retained local polygons are written +to `outlooks`, and run-level day discussions are written to +`outlook_discussions`. The parent run row stores `outlook_count` and +`discussion_count`. + ## Boundaries - Weatherfeeder owns table definitions in `schema.go`. @@ -82,6 +87,7 @@ Child rows use positional indexes to preserve canonical array order: - `alert_index` - `reference_index` - `outlook_index` +- `discussion_index` Required canonical fields are validated before writes are returned: @@ -94,7 +100,10 @@ Required canonical fields are validated before writes are returned: - alert runs require `asOf`, and each alert requires `id`; - outlook runs require `asOf`, and each outlook requires `id`, `provider`, `product`, `day`, `outlookType`, `label`, `validFrom`, `validTo`, `issuedAt`, - `expiresAt`, and `geometry`. + `expiresAt`, `containsLocation: true`, and `geometry`; +- outlook discussions require day `1`, `2`, or `3`; at least one of + `headline`, `summary`, or `discussion`; and no duplicate discussion day in the + same run. Nullable canonical values are converted to SQL nulls by mapper helpers. Observation present-weather raw values and outlook geometry values are stored as diff --git a/docs/operations.md b/docs/operations.md index 08080e3..290083f 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -82,7 +82,7 @@ document bodies as the previous successful poll. Every event passes through normalization first and dedupe second. -Normalizers match raw source schemas and produce canonical `weather.*.v1` +Normalizers match raw source schemas and produce versioned canonical `weather.*` payloads. If an event has no matching normalizer, the normalize processor passes it through unchanged. diff --git a/docs/policy/architecture.md b/docs/policy/architecture.md index a086b6d..d7873de 100644 --- a/docs/policy/architecture.md +++ b/docs/policy/architecture.md @@ -48,6 +48,7 @@ The implementation style is: - Hexagonal boundaries: provider APIs, config loading, scheduling, dispatch, and sinks are external mechanisms around the weather domain model and normalization logic. - Raw-to-canonical flow: sources should fetch and envelope raw provider payloads; normalizers should own provider-to-canonical mapping. +- Location-focused canonical data: canonical weather events represent data relevant to the configured forecast location or configured provider object; `weatherfeeder` is not a national provider-data archive. - Schema-based routing: normalizers match on event schema, not source name or event kind. - Composable registries: source drivers, normalizers, processors, and sinks are assembled explicitly through registries. - Bounded concurrency: scheduling and sink fanout are concurrent, but the application should keep queues, goroutine ownership, logging, and cancellation behavior visible. diff --git a/internal/sinks/postgres/doc.go b/internal/sinks/postgres/doc.go index 8c0d094..22bc38d 100644 --- a/internal/sinks/postgres/doc.go +++ b/internal/sinks/postgres/doc.go @@ -11,7 +11,7 @@ // - weather.forecast_discussion.v1 -> model.WeatherForecastDiscussion // - weather.weather_story.v1 -> model.WeatherStoryRun // - weather.alert.v1 -> model.WeatherAlertRun -// - weather.outlook.v1 -> model.WeatherOutlookRun +// - weather.outlook.v2 -> model.WeatherOutlookRun // // Parent/child relationships: // - observations.event_id -> observation_present_weather.event_id @@ -21,6 +21,7 @@ // - alert_runs.event_id -> alerts.run_event_id // - alerts.(run_event_id, alert_index) -> alert_references.(run_event_id, alert_index) // - outlook_runs.event_id -> outlooks.run_event_id +// - outlook_runs.event_id -> outlook_discussions.run_event_id // // Dedupe and retention behavior: // - Parent primary keys (event_id): observations, forecasts, alert_runs, outlook_runs. @@ -39,6 +40,7 @@ // - alert_references.as_of // - outlook_runs.as_of // - outlooks.as_of +// - outlook_discussions.as_of // // Envelope field mapping (shared parent columns) // @@ -227,6 +229,7 @@ // - as_of TIMESTAMPTZ -> payload.asOf // - issued_at TIMESTAMPTZ NULL -> payload.issuedAt // - outlook_count INTEGER -> len(payload.outlooks) +// - discussion_count INTEGER -> len(payload.discussions) // // 11. outlooks (PK: run_event_id, outlook_index) // @@ -246,14 +249,22 @@ // - issued_at TIMESTAMPTZ -> payload.outlooks[i].issuedAt // - expires_at TIMESTAMPTZ -> payload.outlooks[i].expiresAt // - forecaster TEXT NULL -> payload.outlooks[i].forecaster -// - headline TEXT NULL -> payload.outlooks[i].headline -// - summary TEXT NULL -> payload.outlooks[i].summary -// - discussion TEXT NULL -> payload.outlooks[i].discussion // - source_url TEXT NULL -> payload.outlooks[i].sourceUrl // - image_url TEXT NULL -> payload.outlooks[i].imageUrl // - contains_location BOOLEAN -> payload.outlooks[i].containsLocation // - geometry_json TEXT -> compact JSON payload.outlooks[i].geometry // +// 12. outlook_discussions (PK: run_event_id, discussion_index) +// +// - run_event_id TEXT -> outlook_runs.event_id / payload.discussions[i] +// - discussion_index INTEGER -> i (array position in payload.discussions) +// - as_of TIMESTAMPTZ -> payload.asOf (copied from parent) +// - day INTEGER -> payload.discussions[i].day +// - headline TEXT NULL -> payload.discussions[i].headline +// - summary TEXT NULL -> payload.discussions[i].summary +// - discussion TEXT NULL -> payload.discussions[i].discussion +// - updated_at TIMESTAMPTZ NULL -> payload.discussions[i].updatedAt +// // Reconstructing canonical JSON payloads // // - WeatherObservation: @@ -274,6 +285,7 @@ // ordered by reference_index to rebuild references per alert. // // - WeatherOutlookRun: -// read one row from outlook_runs, then join outlooks by run_event_id ordered -// by outlook_index to rebuild outlooks. +// read one row from outlook_runs, join outlooks by run_event_id ordered by +// outlook_index to rebuild outlooks, then join outlook_discussions by +// run_event_id ordered by discussion_index to rebuild discussions. package postgres diff --git a/internal/sinks/postgres/docs_test.go b/internal/sinks/postgres/docs_test.go new file mode 100644 index 0000000..8a512e4 --- /dev/null +++ b/internal/sinks/postgres/docs_test.go @@ -0,0 +1,33 @@ +package postgres + +import ( + "os" + "strings" + "testing" +) + +func TestDocumentedOutlookDiscussionStorage(t *testing.T) { + for _, path := range []string{ + "../../../docs/integrations/postgres.md", + "../../../docs/internal/postgres-sink.md", + } { + t.Run(path, func(t *testing.T) { + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile(%s) error = %v", path, err) + } + doc := string(raw) + + for _, want := range []string{ + tableOutlookDiscussions, + "discussion_count", + "discussion_index", + "weather.outlook.v2", + } { + if !strings.Contains(doc, want) { + t.Fatalf("%s missing %q", path, want) + } + } + }) + } +} diff --git a/standards/docs_test.go b/standards/docs_test.go index 0ee32b2..67413fd 100644 --- a/standards/docs_test.go +++ b/standards/docs_test.go @@ -98,7 +98,7 @@ func stringConstantsFromFile(t *testing.T, path string, prefix string, skip func func schemaConstantNotInCurrentContract(name string) bool { switch name { - case "SchemaRawOpenWeatherHourlyForecastV1", "SchemaWeatherOutlookV2": + case "SchemaRawOpenWeatherHourlyForecastV1": return true default: return false