57 Commits

Author SHA1 Message Date
a0389ebce8 Added support for Area Forecast Discussions issued by the NWS
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-03-28 16:17:03 -05:00
40f17c9d86 Updates to track upstream feedkit v0.8.2
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-03-28 13:53:54 -05:00
c76088c38c Code cleanup and deduplication pass through weatherfeeder
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
ci/woodpecker/manual/build-image Pipeline was successful
2026-03-28 12:01:07 -05:00
2c1278a70a Moved generic and broadly useful helper functions upstream into feedkit
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-03-28 11:30:20 -05:00
eb27486466 Moved HTTP polling helpers upstream into feedkit, and updated to feedkit v0.8.0
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-03-28 10:02:50 -05:00
de5add59fd Updated default config.yml to include a commented postgres sink example with pruning enabled
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-03-28 08:04:44 -05:00
356c3be648 Feature addition to support narrative forecast updates from the NWS
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-03-27 16:07:12 -05:00
dbaebbbd7a Updates to the nws forecast source and normalizer to separate code specific to hourly forecasts and prepare for upcoming feature addition of daily and narrative forecasts
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-03-27 12:58:23 -05:00
88d5727a84 Simplified the forecast schema
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-03-26 21:35:08 -05:00
129cebd94d Updated the normalized observation schema to remove duplicate and/or unnecessary fields
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-03-17 11:04:51 -05:00
e42f2bc9de Remove incorrect 'internal/' prefix from model package file header comments 2026-03-17 09:46:39 -05:00
9ddcf5e0df Document the PostgreSQL schema contract in doc.go
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-03-17 09:33:07 -05:00
d0b58a4734 Updates to track feedkit v0.7.2 and to add a dedupe processor
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-03-16 18:35:44 -05:00
6cd823f528 Update go.mod
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-03-16 15:37:31 -05:00
84da2bb689 Added a postgres sink implementation.
Some checks failed
ci/woodpecker/push/build-image Pipeline failed
2026-03-16 15:32:46 -05:00
859ee9dd5c Updated go.sum
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-03-16 13:37:49 -05:00
ea113e2dcc Updated processor/normalizer wiring to track Feedkit v0.7.0
Some checks failed
ci/woodpecker/push/build-image Pipeline failed
2026-03-16 13:35:51 -05:00
38bc162918 Updated go.sum
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-03-15 19:26:51 -05:00
eae9568afe Updated source configuration to track Feedkit v0.6.0
Some checks failed
ci/woodpecker/push/build-image Pipeline failed
2026-03-15 19:22:57 -05:00
f464592c56 Updates to accommodate the new upstream version of feedkit (v0.5.0), which now supports both polling sources and streaming sources.
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-02-08 15:05:53 -06:00
123e8ff763 Moved the standards package out of internal/ so it can be imported by downstream consumers.
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-02-08 09:15:07 -06:00
5923592b53 Moved the weatherfeeder model out of internal/ so that downstream consumers can import it directly.
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-02-08 08:56:16 -06:00
c96a6bb78b Remove Go test files from .dockerignore; this will enable the CI system to run tests before building. 2026-02-08 08:55:35 -06:00
cff8c5f593 Merged SCHEMA.md into API.md and made some tweaks to ensure consistency with the underlying domain model.
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-02-07 19:36:50 -06:00
190420e908 Added a SCHEMA.md with espress documentation of the JSON output schemas emitted by weatherfeeder.
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-02-07 17:26:07 -06:00
9663fdfc43 Updated main.go to register the postgres and NATS sinks in addition to the stdout sink.
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
2026-02-07 11:57:54 -06:00
ffd4dfa76d Updated feedkit dependency to v0.4.1.
All checks were successful
ci/woodpecker/manual/build-image Pipeline was successful
2026-02-07 11:37:53 -06:00
d858bc1f31 Updated Dockerfile and Woodpecker pipeline to use harbor.maximumdirect.net proxy cache for upstream images.
All checks were successful
ci/woodpecker/manual/build-image Pipeline was successful
ci/woodpecker/push/build-image Pipeline was successful
2026-02-01 19:39:55 -06:00
2a88c3c5f3 Updated the Woodpecker pipeline to utilize caching.
All checks were successful
ci/woodpecker/manual/build-image Pipeline was successful
2026-02-01 19:04:22 -06:00
33c35b5f4a Update Dockerfile to streamline the weatherfeeder user creation.
All checks were successful
ci/woodpecker/manual/build-image Pipeline was successful
2026-02-01 18:17:12 -06:00
df3f42ef30 Updated the Dockerfile to reflect best practices; removed the dropreplace lines after updates to go.mod.
All checks were successful
ci/woodpecker/manual/build-image Pipeline was successful
2026-02-01 16:59:34 -06:00
e86d4a02e0 Added a go mod tidy step to the Dockerfile build.
All checks were successful
ci/woodpecker/manual/build-image Pipeline was successful
2026-02-01 10:15:50 -06:00
ba95371418 Added a second dropreplace step for ejr/feedkit.
Some checks failed
ci/woodpecker/manual/build-image Pipeline failed
2026-02-01 10:10:53 -06:00
5a92fc0401 Fixed a bug in the Dockerfile with respect to the location of the dropreplace for ejr/feedkit.
Some checks failed
ci/woodpecker/manual/build-image Pipeline failed
2026-02-01 10:01:02 -06:00
f522840fed Updated go.mod to require feedkit v0.4.0.
Some checks failed
ci/woodpecker/manual/build-image Pipeline failed
2026-02-01 09:57:10 -06:00
a46cbe17d1 Updated the Dockerfile to drop the ejr/feedkit replacement and to run go test before building.
Some checks failed
ci/woodpecker/manual/build-image Pipeline failed
2026-02-01 09:47:17 -06:00
7b8b3b98f2 Added a Dockerfile and completely refactored the Woodpecker pipeline.
Some checks failed
ci/woodpecker/manual/build-image Pipeline failed
2026-02-01 09:38:26 -06:00
4600ca053b woodpecker: Update pipeline to allow manual triggering. 2026-02-01 08:10:24 -06:00
88d2385576 woodpecker: allow manual pipeline triggering. 2026-02-01 07:33:01 -06:00
6cdbb29215 woodpecker: refactor pipeline to use kaniko. 2026-02-01 07:22:28 -06:00
62464f449e More woodpecker changes. 2026-01-31 13:56:12 -06:00
db8c66832f Removed privileged: true from the Woodpecker pipeline. 2026-01-31 13:53:27 -06:00
062b12c44f Add CI/CD pipeline configuration for building and publishing weatherfeeder. 2026-01-31 13:50:09 -06:00
da0231b20f Add day/night estimation for NWS observations based on solar elevation and update observation model. 2026-01-22 23:15:17 -06:00
c675045013 Updated documentation and added API.md to document the stable external wire format. 2026-01-17 17:39:18 -06:00
b4a67e208c Updated model and normalizers to map provider “feels like” fields into a single ApparentTemperatureC field. 2026-01-17 10:36:19 -06:00
c12cf91115 normalizers: implemented openmeteo forecast normalizer. 2026-01-17 10:16:50 -06:00
b47f1b2051 sources: added an OpenMeteo forecast source. 2026-01-17 08:00:23 -06:00
b8804d32d2 refactor(normalizers): deduplicate synthetic station ID generation
- Add common SynthStationID helpers for coordinate-based providers
- Use shared helper for Open-Meteo and OpenWeather station ID synthesis
- Require both lat/lon when generating synthetic IDs to avoid misleading defaults
- Remove unused Open-Meteo normalizer wrapper code

This reduces cross-provider duplication while keeping provider-specific
mapping logic explicit and readable.
2026-01-16 22:13:44 -06:00
00e811f8f7 normalizers/nws: add NWS alerts normalizer and canonical alert mapping
- Introduce AlertsNormalizer to convert Raw NWS Alerts (SchemaRawNWSAlertsV1)
  into canonical WeatherAlert runs (SchemaWeatherAlertV1)
- Add minimal NWS alerts response/types to support GeoJSON FeatureCollection parsing
- Map NWS alert properties (event, headline, severity, timing, area, references)
  into model.WeatherAlert with best-effort timestamp handling
- Establish clear AsOf / EffectiveAt policy for alert runs to support stable
  deduplication and snapshot semantics
- Register the new alerts normalizer alongside existing NWS observation and
  forecast normalizers
2026-01-16 21:40:20 -06:00
2eb2d4b90f feat(nws, normalizers): add NWS hourly forecast normalization and enforce canonical float rounding
- Implement full NWS hourly forecast normalizer (raw.nws.hourly.forecast.v1 → weather.forecast.v1)
- Add GeoJSON forecast types and helpers for NWS gridpoint hourly payloads
- Normalize temperatures, winds, humidity, PoP, and infer WMO condition codes from forecast text/icons
- Treat forecast IssuedAt as EffectiveAt for stable, dedupe-friendly event IDs

- Introduce project-wide float rounding at normalization finalization
  - Round all float values in canonical payloads to 2 decimal places
  - Apply consistently across pointers, slices, maps, and nested structs
  - Preserve opaque structs (e.g., time.Time) unchanged

- Add SchemaRawNWSHourlyForecastV1 and align schema matching/comments
- Clean up NWS helper organization and comments
- Update documentation to reflect numeric wire-format and normalization policies

This establishes a complete, deterministic hourly forecast pipeline for NWS
and improves JSON output stability across all canonical weather schemas.
2026-01-16 10:28:32 -06:00
0fcc536885 Updates in preparation for adding forecast sources. 2026-01-16 00:04:37 -06:00
e10ba804ca model: add explicit JSON tags and document canonical payload contract
Add lowerCamelCase JSON tags to canonical model types (observation, forecast,
alert) to stabilize the emitted wire format and make payload structure explicit
for downstream sinks.

Introduce internal/model/doc.go to document these structs as versioned,
schema-governed payloads and clarify compatibility expectations (additive
changes preferred; breaking changes require schema bumps).

No functional behavior changes; this formalizes the canonical output contract
ahead of additional sinks and consumers.
2026-01-15 22:39:37 -06:00
f13f43cf56 refactor(providers): centralize provider-specific parsing and invariants
- Introduce internal/providers/nws with shared timestamp parsing used by both
  NWS sources and normalizers
- Migrate NWS observation source + normalizer to use the shared provider helper
  for consistent RFC3339/RFC3339Nano handling
- Introduce internal/providers/openweather with a shared URL invariant helper
  enforcing units=metric
- Remove duplicated OpenWeather URL validation logic from the observation source
- Align provider layering: move provider “contract/quirk” logic out of
  normalizers and into internal/providers
- Update normalizer and standards documentation to clearly distinguish:
  provider helpers (internal/providers) vs canonical mapping logic
  (internal/normalizers)

This refactor reduces duplication between sources and normalizers, clarifies
layering boundaries, and establishes a scalable pattern for future forecast
and alert implementations.
2026-01-15 20:40:53 -06:00
a341aee5df normalizers: Updated error handling within the JSON helper function. 2026-01-15 20:17:46 -06:00
d8db58c004 sources: standardize Event.ID on Source:EffectiveAt; simplify raw event helper
- Adopt an opinionated Event.ID policy across sources:
  - use upstream-provided ID when available
  - otherwise derive a stable ID from Source:EffectiveAt (RFC3339Nano, UTC)
  - fall back to Source:EmittedAt when EffectiveAt is unavailable
- Add common/id helper to centralize ID selection logic and keep sources consistent
- Simplify common event construction by collapsing SingleRawEventAt/SingleRawEvent
  into a single explicit SingleRawEvent helper (emittedAt passed in)
- Update NWS/Open-Meteo/OpenWeather observation sources to:
  - compute EffectiveAt first
  - generate IDs via the shared helper
  - build envelopes via the unified SingleRawEvent helper
- Improve determinism and dedupe-friendliness without changing schemas or payloads
2026-01-15 19:38:15 -06:00
d9474b5a5b v0.x: add reusable HTTP source spine; fix routing; upstream HTTP transport helper
- fix dispatch route compilation so empty Kinds matches all (nil), not none
- introduce internal/sources/common/HTTPSource to centralize HTTP polling boilerplate:
  - standard cfg parsing (url + user_agent)
  - default HTTP client + Accept/User-Agent headers
  - consistent error wrapping
- refactor observation sources (nws/openmeteo/openweather) to use HTTPSource
- upstream generic HTTP fetch/limits/timeout helper from weatherfeeder to feedkit:
  - move internal/sources/common/http.go -> feedkit/transport/http.go
  - keep behavior: status checks, max-body limit, default timeout
2026-01-15 19:11:58 -06:00
91 changed files with 6944 additions and 1210 deletions

7
.dockerignore Normal file
View File

@@ -0,0 +1,7 @@
.git
.gitignore
**/*.md
dist/
tmp/
.DS_Store

View File

@@ -0,0 +1,17 @@
when:
# Allow both normal runs (push) and UI-triggered runs (manual)
- event: [push, manual]
steps:
- name: build-and-push-image
image: harbor.maximumdirect.net/proxy-dockerhub/woodpeckerci/plugin-kaniko
settings:
registry: harbor.maximumdirect.net
repo: build/weatherfeeder
auto_tag: true
username:
from_secret: HARBOR_ROBOT_USER
password:
from_secret: HARBOR_ROBOT_TOKEN
cache: true
cache_repo: build-cache/weatherfeeder

339
API.md Normal file
View File

@@ -0,0 +1,339 @@
# weatherfeeder API (Wire Contract)
This document defines the stable, consumer-facing JSON contract emitted by weatherfeeder sinks.
weatherfeeder emits **events** encoded as JSON. Each event has:
- an **envelope** (metadata + schema identifier), and
- a **payload** whose shape is determined by `schema`.
Downstream consumers should:
1. parse the event envelope,
2. switch on `schema`, then
3. decode `payload` into the matching schema.
---
## Event envelope
All events are JSON objects with these fields:
| Field | Type | Required | Notes |
|---|---:|:---:|---|
| `id` | string | yes | Stable event identifier. Treat as opaque. |
| `schema` | string | yes | Schema identifier (e.g. `weather.observation.v1`). |
| `source` | string | yes | Provider/source identifier (stable within configuration). |
| `effectiveAt` | string (timestamp) | yes | RFC3339Nano timestamp indicating when this event is effective. |
| `payload` | object | yes | Schema-specific payload (see below). |
### Timestamp format
All timestamps are encoded as JSON strings using Gos `time.Time` JSON encoding (RFC3339Nano).
Examples:
- `"2026-01-17T14:27:00Z"`
- `"2026-01-17T08:27:00-06:00"`
---
## Canonical schemas
weatherfeeder emits four canonical domain schemas:
- `weather.observation.v1`
- `weather.forecast.v1`
- `weather.forecast_discussion.v1`
- `weather.alert.v1`
Each payload is described below using the JSON field names as the contract.
### Raw upstream schemas
weatherfeeder sources also emit provider-specific raw schemas before normalization.
For this feature, the raw source schema is:
- `raw.nws.forecast_discussion.v1`
- payload type: string
- payload contents: exact fetched HTML response body
---
## Shared Conventions
- Timestamps are JSON strings in RFC3339Nano format.
- Optional fields are omitted when unknown (`omitempty` behavior).
- Numeric measurements are normalized to metric units:
- `*C` = Celsius
- `*Kmh` = kilometers/hour
- `*Pa` = Pascals
- `*Meters` = meters
- `*Mm` = millimeters
- `*Percent` = percent (0-100)
- `conditionCode` is a WMO weather interpretation code (`int`).
- Unknown/unmappable is `-1`.
- Downstream consumers should treat unknown codes as “unknown conditions” rather than failing decoding.
- For readability and stability, weatherfeeder rounds floating-point values in canonical payloads to
**4 digits after the decimal** during normalization.
---
## Schema: `weather.observation.v1`
Payload type: `WeatherObservation`
A `WeatherObservation` represents a point-in-time observation for a station/location.
### Fields
| Field | Type | Required | Notes |
|---|---:|:---:|---|
| `stationId` | string | no | Provider station/location identifier |
| `stationName` | string | no | Human station name |
| `timestamp` | timestamp string | yes | Observation timestamp |
| `conditionCode` | int | yes | WMO code (`-1` unknown) |
| `isDay` | bool | no | Day/night hint |
| `textDescription` | string | no | Human-facing short description |
| `temperatureC` | number | no | Celsius |
| `dewpointC` | number | no | Celsius |
| `windDirectionDegrees` | number | no | Degrees |
| `windSpeedKmh` | number | no | km/h |
| `windGustKmh` | number | no | km/h |
| `barometricPressurePa` | number | no | Pascals |
| `visibilityMeters` | number | no | Meters |
| `relativeHumidityPercent` | number | no | Percent |
| `apparentTemperatureC` | number | no | Celsius |
| `presentWeather` | array | no | Provider-specific structured weather fragments |
### Nested: `presentWeather[]`
Each `presentWeather[]` element:
| Field | Type | Required | Notes |
|---|---:|:---:|---|
| `raw` | object | no | Provider-specific JSON object |
---
## Schema: `weather.forecast.v1`
Payload type: `WeatherForecastRun`
A `WeatherForecastRun` is a single issued forecast snapshot for a location and a specific product
(hourly / narrative / daily). The run contains an ordered list of forecast periods.
### `product` values
`product` is one of:
- `"hourly"`
- `"narrative"`
- `"daily"`
### Fields
| Field | Type | Required | Notes |
|---|---:|:---:|---|
| `locationId` | string | no | Provider location identifier |
| `locationName` | string | no | Human name, if available |
| `issuedAt` | string (timestamp) | yes | When this run was generated/issued |
| `updatedAt` | string (timestamp) | no | Optional later update time |
| `product` | string | yes | One of `hourly`, `narrative`, `daily` |
| `latitude` | number | no | Degrees |
| `longitude` | number | no | Degrees |
| `elevationMeters` | number | no | meters |
| `periods` | array | yes | Chronological forecast periods |
### Nested: `periods[]` (`WeatherForecastPeriod`)
A `WeatherForecastPeriod` is valid for `[startTime, endTime)`.
| Field | Type | Required | Units / Notes |
|---|---:|:---:|---|
| `startTime` | string (timestamp) | yes | Period start |
| `endTime` | string (timestamp) | yes | Period end |
| `name` | string | no | Human label (often empty for hourly) |
| `isDay` | bool | no | Day/night hint |
| `conditionCode` | int | yes | WMO code (`-1` for unknown) |
| `textDescription` | string | no | Human-facing short phrase |
| `temperatureC` | number | no | °C |
| `temperatureCMin` | number | no | °C (aggregated products) |
| `temperatureCMax` | number | no | °C (aggregated products) |
| `dewpointC` | number | no | °C |
| `relativeHumidityPercent` | number | no | percent |
| `windDirectionDegrees` | number | no | degrees |
| `windSpeedKmh` | number | no | km/h |
| `windGustKmh` | number | no | km/h |
| `barometricPressurePa` | number | no | Pa |
| `visibilityMeters` | number | no | meters |
| `apparentTemperatureC` | number | no | °C |
| `cloudCoverPercent` | number | no | percent |
| `probabilityOfPrecipitationPercent` | number | no | percent |
| `precipitationAmountMm` | number | no | mm (liquid equivalent) |
| `snowfallDepthMm` | number | no | mm |
| `uvIndex` | number | no | unitless index |
---
## Schema: `weather.alert.v1`
Payload type: `WeatherAlertRun`
A `WeatherAlertRun` is a snapshot of *active* alerts for a location as-of a point in time.
A run may contain zero, one, or many alerts.
### Fields
| Field | Type | Required | Notes |
|---|---:|:---:|---|
| `locationId` | string | no | Provider location identifier |
| `locationName` | string | no | Human name, if available |
| `asOf` | string (timestamp) | yes | When the provider asserted this snapshot is current |
| `latitude` | number | no | Degrees |
| `longitude` | number | no | Degrees |
| `alerts` | array | yes | Active alerts (order provider-dependent) |
### Nested: `alerts[]` (`WeatherAlert`)
| Field | Type | Required | Notes |
|---|---:|:---:|---|
| `id` | string | yes | Provider-stable identifier (often a URL/URI) |
| `event` | string | no | Classification/event label |
| `headline` | string | no | Alert headline |
| `severity` | string | no | Example: Extreme/Severe/Moderate/Minor/Unknown |
| `urgency` | string | no | Example: Immediate/Expected/Future/Past/Unknown |
| `certainty` | string | no | Example: Observed/Likely/Possible/Unlikely/Unknown |
| `status` | string | no | Example: Actual/Exercise/Test/System/Unknown |
| `messageType` | string | no | Example: Alert/Update/Cancel |
| `category` | string | no | Example: Met/Geo/Safety/Rescue/Fire/Health/Env/Transport/Infra/CBRNE/Other |
| `response` | string | no | Example: Shelter/Evacuate/Prepare/Execute/Avoid/Monitor/Assess/AllClear/None |
| `response` | string | no | e.g. Shelter/Evacuate/Prepare/... |
| `description` | string | no | Narrative |
| `instruction` | string | no | What to do |
| `sent` | string (timestamp) | no | Provider-dependent |
| `effective` | string (timestamp) | no | Provider-dependent |
| `onset` | string (timestamp) | no | Provider-dependent |
| `expires` | string (timestamp) | no | Provider-dependent |
| `areaDescription` | string | no | Often a provider string |
| `senderName` | string | no | Provenance |
| `references` | array | no | Related alert references |
### Nested: `references[]` (`AlertReference`)
| Field | Type | Required | Notes |
|---|---:|:---:|---|
| `id` | string | no | Provider reference ID/URI |
| `identifier` | string | no | Provider identifier string, if distinct |
| `sender` | string | no | Sender |
| `sent` | string (timestamp) | no | Timestamp |
---
## Schema: `weather.forecast_discussion.v1`
Payload type: `WeatherForecastDiscussion`
A `WeatherForecastDiscussion` is an issued narrative bulletin for an NWS office.
It is distinct from `weather.forecast.v1`, which is period-based.
### Fields
| Field | Type | Required | Notes |
|---|---:|:---:|---|
| `officeId` | string | no | NWS office identifier, e.g. `LSX` |
| `officeName` | string | no | Human office name |
| `product` | string | yes | Currently `afd` |
| `issuedAt` | string (timestamp) | yes | Bulletin issue time |
| `updatedAt` | string (timestamp) | no | Optional page/update timestamp |
| `keyMessages` | array | no | Ordered key-message bullet list |
| `shortTerm` | object | no | Short-term discussion section |
| `longTerm` | object | no | Long-term discussion section |
### Nested: `shortTerm` / `longTerm`
| Field | Type | Required | Notes |
|---|---:|:---:|---|
| `qualifier` | string | no | Header qualifier such as `(Through Late Sunday Night)` |
| `issuedAt` | string (timestamp) | no | Optional section-local issue time |
| `text` | string | no | Paragraph-preserved prose text |
---
## Compatibility rules
- Consumers **must** ignore unknown fields.
- Producers (weatherfeeder) prefer **additive changes** within a schema version.
- Renames/removals/semantic breaks normally require a **schema version bump** (`weather.*.v2`); pre-1.0 projects may choose in-place changes.
---
## Examples
### Observation event (`weather.observation.v1`)
```json
{
"id": "nws:KSTL:2026-01-17T14:00:00Z",
"schema": "weather.observation.v1",
"source": "nws_observation",
"effectiveAt": "2026-01-17T14:00:00Z",
"payload": {
"stationId": "KSTL",
"timestamp": "2026-01-17T14:00:00Z",
"conditionCode": 1,
"textDescription": "Mainly Sunny",
"temperatureC": 3.25,
"windSpeedKmh": 18.5
}
}
```
### Forecast event (`weather.forecast.v1`)
```json
{
"id": "openmeteo:38.63,-90.20:2026-01-17T13:00:00Z",
"schema": "weather.forecast.v1",
"source": "openmeteo_forecast",
"effectiveAt": "2026-01-17T13:00:00Z",
"payload": {
"locationName": "St. Louis, MO",
"issuedAt": "2026-01-17T13:00:00Z",
"product": "hourly",
"latitude": 38.63,
"longitude": -90.2,
"periods": [
{
"startTime": "2026-01-17T14:00:00Z",
"endTime": "2026-01-17T15:00:00Z",
"conditionCode": 2,
"textDescription": "Partly Cloudy",
"temperatureC": 3.5,
"probabilityOfPrecipitationPercent": 10
}
]
}
}
```
### Alert event (`weather.alert.v1`)
```json
{
"id": "nws:alerts:2026-01-17T14:10:00Z",
"schema": "weather.alert.v1",
"source": "nws_alerts",
"effectiveAt": "2026-01-17T14:10:00Z",
"payload": {
"asOf": "2026-01-17T14:05:00Z",
"alerts": [
{
"id": "https://api.weather.gov/alerts/abc123",
"event": "Winter Weather Advisory",
"headline": "Winter Weather Advisory issued January 17 at 8:05AM CST",
"severity": "Moderate",
"description": "Mixed precipitation expected...",
"expires": "2026-01-18T06:00:00Z"
}
]
}
}
```

78
Dockerfile Normal file
View File

@@ -0,0 +1,78 @@
# syntax=docker/dockerfile:1.6
ARG GO_VERSION=1.25
############################
# Build stage
############################
FROM harbor.maximumdirect.net/proxy-dockerhub/golang:${GO_VERSION}-bookworm AS build
WORKDIR /src
# Install baseline packages
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates tzdata git build-essential \
&& rm -rf /var/lib/apt/lists/*
# Cache dependencies first
COPY go.mod go.sum ./
RUN --mount=type=cache,target=/go/pkg/mod \
go mod download
# Copy the rest of the source
COPY . .
# Ensure go.sum is complete after dropping the replace
RUN --mount=type=cache,target=/go/pkg/mod \
--mount=type=cache,target=/root/.cache/go-build
# Default to a static build (no CGO)
# If errors, can build with: --build-arg CGO_ENABLED=1
ARG CGO_ENABLED=0
ARG TARGETOS=linux
ARG TARGETARCH=amd64
ENV CGO_ENABLED=${CGO_ENABLED} \
GOOS=${TARGETOS} \
GOARCH=${TARGETARCH}
# Run tests before building the final binary
RUN --mount=type=cache,target=/go/pkg/mod \
--mount=type=cache,target=/root/.cache/go-build \
go test ./...
# Build the cmd entrypoint
RUN --mount=type=cache,target=/root/.cache/go-build \
go build \
-trimpath \
-ldflags="-s -w" \
-o /out/weatherfeeder \
./cmd/weatherfeeder
############################
# Runtime stage
############################
FROM harbor.maximumdirect.net/proxy-dockerhub/debian:bookworm-slim AS runtime
# Install runtime necessities
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates tzdata curl \
&& rm -rf /var/lib/apt/lists/*
# Define /weatherfeeder as the working directory
WORKDIR /weatherfeeder
# Create an unprivileged user
RUN useradd \
--uid 10001 \
--no-create-home \
--shell /usr/sbin/nologin \
weatherfeeder
# Copy the binary
COPY --chown=weatherfeeder:weatherfeeder --from=build /out/weatherfeeder /weatherfeeder/weatherfeeder
USER weatherfeeder
# The application expects config.yml in the same directory as the binary
ENTRYPOINT ["/weatherfeeder/weatherfeeder"]

View File

@@ -1,3 +1,35 @@
# weatherfeeder # weatherfeeder
A small daemon to poll weather observations, alerts, and forecasts from a variety of sources. weatherfeeder is a small daemon that polls weather observations, forecasts, and alerts from multiple upstream
providers, normalizes them into a provider-independent format, and emits them to a sink.
Today, the only implemented sink is `stdout`, which prints JSON-encoded events.
## What weatherfeeder emits
weatherfeeder emits **feed events** encoded as JSON. Each event includes a schema identifier and a payload.
Downstream consumers should key off the `schema` value and decode the `payload` accordingly.
Canonical domain schemas emitted after normalization:
- `weather.observation.v1``WeatherObservation`
- `weather.forecast.v1``WeatherForecastRun`
- `weather.forecast_discussion.v1``WeatherForecastDiscussion`
- `weather.alert.v1``WeatherAlertRun`
For the complete wire contract (event envelope + payload schemas, fields, units, and compatibility rules), see:
- **API.md**
## Upstream providers (current MVP)
- NWS: observations, hourly forecasts, narrative forecasts, forecast discussions, alerts
- Open-Meteo: observations, hourly forecasts
- OpenWeather: observations
## Versioning & compatibility
The JSON field names on canonical payload types are treated as part of the wire contract.
Additive changes are preferred. Renames/removals require a schema version bump.
See **API.md** for details.

View File

@@ -1,58 +1,115 @@
--- ---
sources: sources:
- name: NWSObservationKSTL - name: NWSObservationKSTL
kind: observation mode: poll
kinds: ["observation"]
driver: nws_observation driver: nws_observation
every: 12m every: 10m
params: params:
url: "https://api.weather.gov/stations/KSTL/observations/latest" url: "https://api.weather.gov/stations/KSTL/observations/latest"
user_agent: "HomeOps (eric@maximumdirect.net)" user_agent: "HomeOps (eric@maximumdirect.net)"
- name: OpenMeteoObservation # - name: OpenMeteoObservation
kind: observation # mode: poll
driver: openmeteo_observation # kinds: ["observation"]
every: 12m # driver: openmeteo_observation
params: # every: 10m
url: "https://api.open-meteo.com/v1/forecast?latitude=38.6239&longitude=-90.3571&current=temperature_2m,relative_humidity_2m,weather_code,wind_speed_10m,wind_direction_10m,precipitation,surface_pressure,rain,showers,snowfall,cloud_cover,apparent_temperature,is_day,wind_gusts_10m,pressure_msl&forecast_days=1" # params:
user_agent: "HomeOps (eric@maximumdirect.net)" # url: "https://api.open-meteo.com/v1/forecast?latitude=38.6239&longitude=-90.3571&current=temperature_2m,relative_humidity_2m,weather_code,wind_speed_10m,wind_direction_10m,precipitation,surface_pressure,rain,showers,snowfall,cloud_cover,apparent_temperature,is_day,wind_gusts_10m,pressure_msl"
# user_agent: "HomeOps (eric@maximumdirect.net)"
- name: OpenWeatherObservation # - name: OpenWeatherObservation
kind: observation # mode: poll
driver: openweather_observation # kinds: ["observation"]
every: 12m # driver: openweather_observation
params: # every: 10m
url: "https://api.openweathermap.org/data/2.5/weather?lat=38.6239&lon=-90.3571&appid=c954f2566cb7ccb56b43737b52e88fc6&units=metric" # params:
user_agent: "HomeOps (eric@maximumdirect.net)" # url: "https://api.openweathermap.org/data/2.5/weather?lat=38.6239&lon=-90.3571&appid=c954f2566cb7ccb56b43737b52e88fc6&units=metric"
# user_agent: "HomeOps (eric@maximumdirect.net)"
# - name: NWSObservationKSUS # - name: NWSObservationKSUS
# kind: observation # mode: poll
# kinds: ["observation"]
# driver: nws_observation # driver: nws_observation
# every: 18s # every: 10m
# params: # params:
# url: "https://api.weather.gov/stations/KSUS/observations/latest" # url: "https://api.weather.gov/stations/KSUS/observations/latest"
# user_agent: "HomeOps (eric@maximumdirect.net)" # user_agent: "HomeOps (eric@maximumdirect.net)"
# - name: NWSObservationKCPS # - name: NWSObservationKCPS
# kind: observation # mode: poll
# kinds: ["observation"]
# driver: nws_observation # driver: nws_observation
# every: 12m # every: 10m
# params: # params:
# url: "https://api.weather.gov/stations/KCPS/observations/latest" # url: "https://api.weather.gov/stations/KCPS/observations/latest"
# user_agent: "HomeOps (eric@maximumdirect.net)" # user_agent: "HomeOps (eric@maximumdirect.net)"
# - name: NWSAlertsSTL - name: NWSHourlyForecastSTL
# kind: alert mode: poll
# driver: nws_alerts kinds: ["forecast"]
# every: 1m driver: nws_forecast_hourly
# params: every: 45m
# url: "https://api.weather.gov/alerts?point=38.6239,-90.3571&limit=500" params:
# user_agent: "HomeOps (eric@maximumdirect.net)" url: "https://api.weather.gov/gridpoints/LSX/90,74/forecast/hourly"
user_agent: "HomeOps (eric@maximumdirect.net)"
- name: NWSNarrativeForecastSTL
mode: poll
kinds: ["forecast"]
driver: nws_forecast_narrative
every: 45m
params:
url: "https://api.weather.gov/gridpoints/LSX/90,74/forecast?units=us"
user_agent: "HomeOps (eric@maximumdirect.net)"
- name: NWSForecastDiscussionSTL
mode: poll
kinds: ["forecast_discussion"]
driver: nws_forecast_discussion
every: 30m
params:
url: "https://forecast.weather.gov/product.php?site=LSX&issuedby=LSX&product=AFD&format=TXT&version=1&glossary=0"
user_agent: "HomeOps (eric@maximumdirect.net)"
- name: OpenMeteoHourlyForecastSTL
mode: poll
kinds: ["forecast"]
driver: openmeteo_forecast
every: 60m
params:
url: https://api.open-meteo.com/v1/forecast?latitude=38.6239&longitude=-90.3571&hourly=temperature_2m,relative_humidity_2m,dew_point_2m,apparent_temperature,precipitation_probability,precipitation,snowfall,weather_code,surface_pressure,wind_speed_10m,wind_direction_10m&forecast_days=3
user_agent: "HomeOps (eric@maximumdirect.net)"
- name: NWSAlertsSTL
mode: poll
kinds: ["alert"]
driver: nws_alerts
every: 1m
params:
url: "https://api.weather.gov/alerts?point=38.6239,-90.3571&limit=20"
user_agent: "HomeOps (eric@maximumdirect.net)"
sinks: sinks:
- name: stdout - name: stdout
driver: stdout driver: stdout
params: {} params: {}
- name: nats_weatherfeeder
driver: nats
params:
url: nats://nats:4222
subject: weatherfeeder
# - name: pg_weatherfeeder
# driver: postgres
# params:
# uri: postgres://weatherdb:5432/weatherdb?sslmode=disable
# username: weatherdb
# password: weatherdb
# prune: 3d
# # Prunes rows older than now-3d on each write transaction.
# - name: logfile # - name: logfile
# driver: file # driver: file
# params: # params:
@@ -60,7 +117,13 @@ sinks:
routes: routes:
- sink: stdout - sink: stdout
kinds: ["observation"] kinds: ["observation", "forecast", "forecast_discussion", "alert"]
- sink: nats_weatherfeeder
kinds: ["observation", "forecast", "forecast_discussion", "alert"]
# - sink: pg_weatherfeeder
# kinds: ["observation", "forecast", "forecast_discussion", "alert"]
# - sink: logfile # - sink: logfile
# kinds: ["observation", "alert", "forecast"] # kinds: ["observation", "alert", "forecast", "forecast_discussion"]

View File

@@ -3,27 +3,29 @@ package main
import ( import (
"context" "context"
"errors" "errors"
"fmt"
"log" "log"
"os" "os"
"os/signal" "os/signal"
"strings"
"syscall" "syscall"
"time"
"gitea.maximumdirect.net/ejr/feedkit/config" "gitea.maximumdirect.net/ejr/feedkit/config"
fkdispatch "gitea.maximumdirect.net/ejr/feedkit/dispatch" fkdispatch "gitea.maximumdirect.net/ejr/feedkit/dispatch"
fkevent "gitea.maximumdirect.net/ejr/feedkit/event" fkevent "gitea.maximumdirect.net/ejr/feedkit/event"
fknormalize "gitea.maximumdirect.net/ejr/feedkit/normalize"
fkpipeline "gitea.maximumdirect.net/ejr/feedkit/pipeline" fkpipeline "gitea.maximumdirect.net/ejr/feedkit/pipeline"
fkprocessors "gitea.maximumdirect.net/ejr/feedkit/processors"
fkdedupe "gitea.maximumdirect.net/ejr/feedkit/processors/dedupe"
fknormalize "gitea.maximumdirect.net/ejr/feedkit/processors/normalize"
fkscheduler "gitea.maximumdirect.net/ejr/feedkit/scheduler" fkscheduler "gitea.maximumdirect.net/ejr/feedkit/scheduler"
fksinks "gitea.maximumdirect.net/ejr/feedkit/sinks" fksinks "gitea.maximumdirect.net/ejr/feedkit/sinks"
fksources "gitea.maximumdirect.net/ejr/feedkit/sources" fksources "gitea.maximumdirect.net/ejr/feedkit/sources"
wfnormalizers "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers" wfnormalizers "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers"
wfpgsink "gitea.maximumdirect.net/ejr/weatherfeeder/internal/sinks/postgres"
wfsources "gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources" wfsources "gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources"
) )
const dedupeMaxEntries = 2048
func main() { func main() {
log.SetFlags(log.LstdFlags | log.Lmicroseconds) log.SetFlags(log.LstdFlags | log.Lmicroseconds)
@@ -40,36 +42,34 @@ func main() {
srcReg := fksources.NewRegistry() srcReg := fksources.NewRegistry()
wfsources.RegisterBuiltins(srcReg) wfsources.RegisterBuiltins(srcReg)
// Minimal sink set to compile: stdout only. // Compile stdout, Postgres, and NATS sinks for weatherfeeder. The former is useful for debugging and the latter are the main intended outputs.
sinkReg := fksinks.NewRegistry() sinkReg := fksinks.NewRegistry()
sinkReg.Register("stdout", func(cfg config.SinkConfig) (fksinks.Sink, error) { fksinks.RegisterBuiltins(sinkReg)
return fksinks.NewStdoutSink(cfg.Name), nil sinkReg.Register("postgres", fksinks.PostgresFactory(wfpgsink.PostgresSchema()))
})
// --- Build sources into scheduler jobs --- // --- Build sources into scheduler jobs ---
var jobs []fkscheduler.Job var jobs []fkscheduler.Job
for i, sc := range cfg.Sources { for i, sc := range cfg.Sources {
src, err := srcReg.Build(sc) in, err := srcReg.BuildInput(sc) // may be polling or streaming
if err != nil { if err != nil {
log.Fatalf("build source failed (sources[%d] name=%q driver=%q): %v", i, sc.Name, sc.Driver, err) log.Fatalf("build source failed (sources[%d] name=%q driver=%q): %v", i, sc.Name, sc.Driver, err)
} }
// Optional safety: if config.kind is set, ensure it matches the source.Kind(). if err := fksources.ValidateExpectedKinds(sc, in); err != nil {
if strings.TrimSpace(sc.Kind) != "" { log.Fatalf("source expected kinds validation failed (sources[%d] name=%q driver=%q): %v", i, sc.Name, sc.Driver, err)
expectedKind, err := fkevent.ParseKind(sc.Kind)
if err != nil {
log.Fatalf("invalid kind in config (sources[%d] name=%q kind=%q): %v", i, sc.Name, sc.Kind, err)
}
if src.Kind() != expectedKind {
log.Fatalf(
"source kind mismatch (sources[%d] name=%q driver=%q): config kind=%q but driver emits kind=%q",
i, sc.Name, sc.Driver, expectedKind, src.Kind(),
)
}
} }
// If this is a polling source, every is required.
if _, ok := in.(fksources.PollSource); ok && sc.Every.Duration <= 0 {
log.Fatalf(
"polling source missing/invalid interval (sources[%d] name=%q driver=%q): sources[].every must be > 0",
i, sc.Name, sc.Driver,
)
}
// For stream sources, Every is ignored; it is fine if omitted/zero.
jobs = append(jobs, fkscheduler.Job{ jobs = append(jobs, fkscheduler.Job{
Source: src, Source: in,
Every: sc.Every.Duration, Every: sc.Every.Duration,
}) })
} }
@@ -85,27 +85,35 @@ func main() {
} }
// --- Compile routes --- // --- Compile routes ---
routes, err := compileRoutes(cfg, builtSinks) routes, err := fkdispatch.CompileRoutes(cfg)
if err != nil { if err != nil {
log.Fatalf("compile routes failed: %v", err) log.Fatalf("compile routes failed: %v", err)
} }
events := make(chan fkevent.Event, 256) events := make(chan fkevent.Event, 256)
// --- Normalization (optional) --- // --- Processors ---
// //
// We install feedkit's normalize.Processor even before any normalizers exist. // We install feedkit's processors/normalize.Processor even before any normalizers exist.
// With an empty registry and RequireMatch=false, this is a no-op passthrough. // With an empty normalizer list and RequireMatch=false, this is a no-op passthrough.
// It will begin transforming events as soon as: // It will begin transforming events as soon as:
// 1) sources emit raw schemas (raw.*), and // 1) sources emit raw schemas (raw.*), and
// 2) matching normalizers are registered. // 2) matching normalizers are registered.
normReg := &fknormalize.Registry{} normalizers := wfnormalizers.RegisterBuiltins(nil)
wfnormalizers.RegisterBuiltins(normReg)
procReg := fkprocessors.NewRegistry()
procReg.Register("normalize", func() (fkprocessors.Processor, error) {
return fknormalize.NewProcessor(normalizers, false), nil
})
procReg.Register("dedupe", fkdedupe.Factory(dedupeMaxEntries))
chain, err := procReg.BuildChain([]string{"normalize", "dedupe"})
if err != nil {
log.Fatalf("build processor chain failed: %v", err)
}
pl := &fkpipeline.Pipeline{ pl := &fkpipeline.Pipeline{
Processors: []fkpipeline.Processor{ Processors: chain,
fknormalize.Processor{Registry: normReg},
},
} }
s := &fkscheduler.Scheduler{ s := &fkscheduler.Scheduler{
@@ -138,55 +146,6 @@ func main() {
log.Printf("shutdown complete") log.Printf("shutdown complete")
} }
func compileRoutes(cfg *config.Config, builtSinks map[string]fksinks.Sink) ([]fkdispatch.Route, error) {
if len(cfg.Routes) == 0 {
return defaultRoutes(builtSinks), nil
}
var routes []fkdispatch.Route
for i, r := range cfg.Routes {
if strings.TrimSpace(r.Sink) == "" {
return nil, fmt.Errorf("routes[%d].sink is empty", i)
}
if _, ok := builtSinks[r.Sink]; !ok {
return nil, fmt.Errorf("routes[%d].sink references unknown sink %q", i, r.Sink)
}
kinds := map[fkevent.Kind]bool{}
for j, k := range r.Kinds {
kind, err := fkevent.ParseKind(k)
if err != nil {
return nil, fmt.Errorf("routes[%d].kinds[%d]: %w", i, j, err)
}
kinds[kind] = true
}
routes = append(routes, fkdispatch.Route{
SinkName: r.Sink,
Kinds: kinds,
})
}
return routes, nil
}
func defaultRoutes(builtSinks map[string]fksinks.Sink) []fkdispatch.Route {
// nil Kinds means "match all kinds" by convention
var allKinds map[fkevent.Kind]bool = nil
routes := make([]fkdispatch.Route, 0, len(builtSinks))
for name := range builtSinks {
routes = append(routes, fkdispatch.Route{
SinkName: name,
Kinds: allKinds,
})
}
return routes
}
func isContextShutdown(err error) bool { func isContextShutdown(err error) bool {
return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded)
} }
// keep time imported (mirrors your previous main.go defensive trick)
var _ = time.Second

View File

@@ -0,0 +1,167 @@
package main
import (
"context"
"reflect"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/ejr/feedkit/config"
fkevent "gitea.maximumdirect.net/ejr/feedkit/event"
fkpipeline "gitea.maximumdirect.net/ejr/feedkit/pipeline"
fkprocessors "gitea.maximumdirect.net/ejr/feedkit/processors"
fkdedupe "gitea.maximumdirect.net/ejr/feedkit/processors/dedupe"
fknormalize "gitea.maximumdirect.net/ejr/feedkit/processors/normalize"
fksources "gitea.maximumdirect.net/ejr/feedkit/sources"
wfnormalizers "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers"
)
type testInput struct {
name string
}
func (s testInput) Name() string { return s.name }
type testKindsSource struct {
testInput
kinds []fkevent.Kind
}
func (s testKindsSource) Kinds() []fkevent.Kind { return s.kinds }
func TestValidateSourceExpectedKindsSubsetAllowed(t *testing.T) {
sc := config.SourceConfig{Kinds: []string{"observation"}}
in := testKindsSource{
testInput: testInput{name: "test"},
kinds: []fkevent.Kind{"observation", "forecast"},
}
if err := fksources.ValidateExpectedKinds(sc, in); err != nil {
t.Fatalf("ValidateExpectedKinds() unexpected error: %v", err)
}
}
func TestValidateSourceExpectedKindsMismatchFails(t *testing.T) {
sc := config.SourceConfig{Kinds: []string{"alert"}}
in := testKindsSource{
testInput: testInput{name: "test"},
kinds: []fkevent.Kind{"observation", "forecast"},
}
err := fksources.ValidateExpectedKinds(sc, in)
if err == nil {
t.Fatalf("ValidateExpectedKinds() expected mismatch error, got nil")
}
if !strings.Contains(err.Error(), "configured expected kind") {
t.Fatalf("ValidateExpectedKinds() error %q does not include expected message", err)
}
}
func TestValidateSourceExpectedKindsNoMetadataSkipsCheck(t *testing.T) {
sc := config.SourceConfig{Kinds: []string{"alert"}}
in := testInput{name: "test"}
if err := fksources.ValidateExpectedKinds(sc, in); err != nil {
t.Fatalf("ValidateExpectedKinds() unexpected error: %v", err)
}
}
func TestExampleConfigLoads(t *testing.T) {
if _, err := config.Load("config.yml"); err != nil {
t.Fatalf("config.Load(config.yml) unexpected error: %v", err)
}
}
func TestProcessorRegistryBuildsNormalizeThenDedupeChain(t *testing.T) {
chain, err := buildProcessorChainForTests()
if err != nil {
t.Fatalf("BuildChain() unexpected error: %v", err)
}
if len(chain) != 2 {
t.Fatalf("BuildChain() expected 2 processors, got %d", len(chain))
}
pl := &fkpipeline.Pipeline{Processors: chain}
if len(pl.Processors) != 2 {
t.Fatalf("pipeline expected 2 processors, got %d", len(pl.Processors))
}
}
func TestNormalizeNoMatchPassThrough(t *testing.T) {
chain, err := buildProcessorChainForTests()
if err != nil {
t.Fatalf("BuildChain() unexpected error: %v", err)
}
pl := &fkpipeline.Pipeline{Processors: chain}
in := fkevent.Event{
ID: "evt-no-match",
Kind: fkevent.Kind("observation"),
Source: "test",
EmittedAt: time.Now().UTC(),
Schema: "raw.weatherfeeder.unknown.v1",
Payload: map[string]any{
"ok": true,
},
}
out, err := pl.Process(context.Background(), in)
if err != nil {
t.Fatalf("Pipeline.Process() unexpected error: %v", err)
}
if out == nil {
t.Fatalf("Pipeline.Process() returned nil output")
}
if !reflect.DeepEqual(*out, in) {
t.Fatalf("Pipeline.Process() expected passthrough output, got %#v", *out)
}
}
func TestDedupeDropsSecondEventWithSameID(t *testing.T) {
chain, err := buildProcessorChainForTests()
if err != nil {
t.Fatalf("BuildChain() unexpected error: %v", err)
}
pl := &fkpipeline.Pipeline{Processors: chain}
in := fkevent.Event{
ID: "evt-dedupe-1",
Kind: fkevent.Kind("observation"),
Source: "test",
EmittedAt: time.Now().UTC(),
Schema: "raw.weatherfeeder.unknown.v1",
Payload: map[string]any{
"ok": true,
},
}
first, err := pl.Process(context.Background(), in)
if err != nil {
t.Fatalf("first Pipeline.Process() unexpected error: %v", err)
}
if first == nil {
t.Fatalf("first Pipeline.Process() unexpectedly dropped event")
}
second, err := pl.Process(context.Background(), in)
if err != nil {
t.Fatalf("second Pipeline.Process() unexpected error: %v", err)
}
if second != nil {
t.Fatalf("second Pipeline.Process() expected dedupe drop, got %#v", *second)
}
}
func buildProcessorChainForTests() ([]fkprocessors.Processor, error) {
normalizers := wfnormalizers.RegisterBuiltins(nil)
procReg := fkprocessors.NewRegistry()
procReg.Register("normalize", func() (fkprocessors.Processor, error) {
return fknormalize.NewProcessor(normalizers, false), nil
})
procReg.Register("dedupe", fkdedupe.Factory(dedupeMaxEntries))
return procReg.BuildChain([]string{"normalize", "dedupe"})
}

17
go.mod
View File

@@ -1,9 +1,16 @@
module gitea.maximumdirect.net/ejr/weatherfeeder module gitea.maximumdirect.net/ejr/weatherfeeder
go 1.22 go 1.25
require gitea.maximumdirect.net/ejr/feedkit v0.0.0 require gitea.maximumdirect.net/ejr/feedkit v0.8.2
require gopkg.in/yaml.v3 v3.0.1 // indirect require (
github.com/klauspost/compress v1.17.2 // indirect
replace gitea.maximumdirect.net/ejr/feedkit => ../feedkit github.com/lib/pq v1.10.9 // indirect
github.com/nats-io/nats.go v1.34.0 // indirect
github.com/nats-io/nkeys v0.4.7 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.18.0 // indirect
golang.org/x/sys v0.16.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

16
go.sum
View File

@@ -1,3 +1,19 @@
gitea.maximumdirect.net/ejr/feedkit v0.8.2 h1:6AMxNacfqJ8SXQhFAUMW3LgiVxixs50tf+S8Q9Ivm+Y=
gitea.maximumdirect.net/ejr/feedkit v0.8.2/go.mod h1:U6xC9xZLN3cL4yi7YBVyzGoHYRLJXusFCAKlj2kdYYQ=
github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/nats-io/nats.go v1.34.0 h1:fnxnPCNiwIG5w08rlMcEKTUw4AV/nKyGCOJE8TdhSPk=
github.com/nats-io/nats.go v1.34.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

View File

@@ -1,23 +0,0 @@
package model
import "time"
// Placeholder for NWS alerts (GeoJSON feature properties are rich).
type WeatherAlert struct {
ID string
Event string
Headline string
Description string
Instruction string
Severity string
Urgency string
Certainty string
Sent *time.Time
Effective *time.Time
Expires *time.Time
Areas []string
}

View File

@@ -1,17 +0,0 @@
package model
import "time"
// WeatherForecast identity fields (as you described).
type WeatherForecast struct {
IssuedBy string // e.g. "NWS"
IssuedAt time.Time // when forecast product was issued
ForecastType string // e.g. "hourly", "daily"
ForecastStart time.Time // start of the applicable forecast period
// TODO: Youll likely want ForecastEnd too.
// TODO: Add meteorological fields you care about.
// Temperature, precip probability, wind, etc.
// Decide if you want a single "period" model or an array of periods.
}

View File

@@ -1,72 +0,0 @@
package model
import "time"
type WeatherObservation struct {
// Identity / metadata
StationID string
StationName string
Timestamp time.Time
// Canonical internal representation (provider-independent).
//
// ConditionCode should be populated by all sources. ConditionText should be the
// canonical human-readable string derived from the WMO code (not the provider's
// original wording).
//
// IsDay is optional; some providers supply a day/night flag (e.g., Open-Meteo),
// while others may not (e.g., NWS observations). When unknown, it can be nil.
ConditionCode WMOCode
ConditionText string
IsDay *bool
// Provider-specific “evidence” for troubleshooting mapping and drift.
//
// This is intentionally limited: it is not intended to be used downstream for
// business logic. Downstream logic should rely on ConditionCode / ConditionText.
ProviderRawDescription string
// Human-facing (legacy / transitional)
//
// TextDescription currently carries provider text in existing drivers.
// As we transition to WMO-based normalization, downstream presentation should
// switch to using ConditionText. After migration, this may be removed or repurposed.
TextDescription string
// Provider-specific (legacy / transitional)
//
// IconURL is not part of the canonical internal vocabulary. It's retained only
// because current sources populate it; it is not required for downstream systems.
IconURL string
// Core measurements (nullable)
TemperatureC *float64
DewpointC *float64
WindDirectionDegrees *float64
WindSpeedKmh *float64
WindGustKmh *float64
BarometricPressurePa *float64
SeaLevelPressurePa *float64
VisibilityMeters *float64
RelativeHumidityPercent *float64
WindChillC *float64
HeatIndexC *float64
ElevationMeters *float64
RawMessage string
PresentWeather []PresentWeather
CloudLayers []CloudLayer
}
type CloudLayer struct {
BaseMeters *float64
Amount string
}
type PresentWeather struct {
Raw map[string]any
}

View File

@@ -2,13 +2,19 @@
package normalizers package normalizers
import ( import (
fknormalize "gitea.maximumdirect.net/ejr/feedkit/normalize" fknormalize "gitea.maximumdirect.net/ejr/feedkit/processors/normalize"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/nws" "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/nws"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openmeteo" "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openmeteo"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openweather" "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openweather"
) )
var builtinRegistrations = []func([]fknormalize.Normalizer) []fknormalize.Normalizer{
nws.Register,
openmeteo.Register,
openweather.Register,
}
// RegisterBuiltins registers all normalizers shipped with this binary. // RegisterBuiltins registers all normalizers shipped with this binary.
// //
// This mirrors internal/sources.RegisterBuiltins, but note the selection model: // This mirrors internal/sources.RegisterBuiltins, but note the selection model:
@@ -16,22 +22,20 @@ import (
// - sources are built by name (cfg.Driver -> factory) // - sources are built by name (cfg.Driver -> factory)
// - normalizers are selected by Match() (event.Schema -> first match wins) // - normalizers are selected by Match() (event.Schema -> first match wins)
// //
// Registration order matters because feedkit normalize.Registry is first match wins. // Registration order matters because feedkit normalize.Processor is "first match wins".
// In weatherfeeder we avoid ambiguity by matching strictly on schema constants, but // In weatherfeeder we avoid ambiguity by matching strictly on schema constants, but
// we still keep ordering stable as a best practice. // we still keep ordering stable as a best practice.
// func RegisterBuiltins(in []fknormalize.Normalizer) []fknormalize.Normalizer {
// If reg is nil, this function is a no-op. out := in
func RegisterBuiltins(reg *fknormalize.Registry) {
if reg == nil {
return
}
// Keep this intentionally boring: delegate registration to provider subpackages // Keep this intentionally boring: delegate registration to provider subpackages
// so main.go stays clean and each provider owns its own mapping logic. // so main.go stays clean and each provider owns its own mapping logic.
// //
// Order here should be stable across releases to reduce surprises when adding // Order here should be stable across releases to reduce surprises when adding
// new normalizers. // new normalizers.
nws.Register(reg) for _, register := range builtinRegistrations {
openmeteo.Register(reg) out = register(out)
openweather.Register(reg) }
return out
} }

View File

@@ -0,0 +1,43 @@
package normalizers
import (
"reflect"
"testing"
fknormalize "gitea.maximumdirect.net/ejr/feedkit/processors/normalize"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/nws"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openmeteo"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/openweather"
)
func TestRegisterBuiltinsOrder(t *testing.T) {
got := RegisterBuiltins(nil)
if len(got) == 0 {
t.Fatalf("RegisterBuiltins() returned no normalizers")
}
want := []fknormalize.Normalizer{
nws.ObservationNormalizer{},
nws.ForecastNormalizer{},
nws.ForecastDiscussionNormalizer{},
nws.AlertsNormalizer{},
openmeteo.ObservationNormalizer{},
openmeteo.ForecastNormalizer{},
openweather.ObservationNormalizer{},
}
if len(got) != len(want) {
t.Fatalf("RegisterBuiltins() expected %d normalizers, got %d", len(want), len(got))
}
for i := range want {
if reflect.TypeOf(got[i]) != reflect.TypeOf(want[i]) {
t.Fatalf(
"RegisterBuiltins() order mismatch at index %d: got %T, want %T",
i,
got[i],
want[i],
)
}
}
}

View File

@@ -11,39 +11,9 @@
// - minimal abstractions (prefer straightforward functions) // - minimal abstractions (prefer straightforward functions)
// - easy to unit test // - easy to unit test
// //
// What belongs here // Numeric wire policy
// ----------------- // -------------------
// Put code in internal/normalizers/common when it is: // Canonical payloads are intended for sinks/serialization. To keep output stable and readable,
// // weatherfeeder rounds floating-point values in canonical payloads to a small, fixed precision
// - potentially reusable by more than one provider // at finalization time (see round.go).
// - provider-agnostic (no NWS/OpenWeather/Open-Meteo specific assumptions)
// - stable, small, and readable
//
// Typical examples:
// - unit conversion helpers (°F <-> °C, m/s <-> km/h, hPa <-> Pa, etc.)
// - json.RawMessage payload extraction helpers (with good error messages)
// - shared parsing helpers (timestamps, simple numeric coercions)
// - generic fallbacks (e.g., mapping a human text description into a coarse canonical code),
// so long as the logic truly applies across providers
//
// What does NOT belong here
// -------------------------
// Do NOT put the following in this package:
//
// - Normalizer implementations (types that satisfy feedkit/normalize.Normalizer)
// - provider-specific JSON structs or mapping logic (put those under
// internal/normalizers/<provider>/)
// - network or filesystem I/O (sources fetch; normalizers transform)
// - code that depends on event.Source naming, config fields, or driver-specific params
//
// Style and API guidelines
// ------------------------
// - Prefer small, single-purpose functions.
// - Keep function names explicit (avoid clever generic “DoThing” helpers).
// - Return typed errors with context (include schema/field names where helpful).
// - Keep dependencies minimal: standard library + weatherfeeder packages only.
// - Add unit tests for any non-trivial logic (especially parsing and fallbacks).
//
// Keeping this clean matters: common is shared by all providers, so complexity here
// multiplies across the project.
package common package common

View File

@@ -5,6 +5,7 @@ import (
"time" "time"
"gitea.maximumdirect.net/ejr/feedkit/event" "gitea.maximumdirect.net/ejr/feedkit/event"
fknormalize "gitea.maximumdirect.net/ejr/feedkit/processors/normalize"
) )
// Finalize builds the output event envelope by copying the input and applying the // Finalize builds the output event envelope by copying the input and applying the
@@ -14,19 +15,9 @@ import (
// - ID/Kind/Source/EmittedAt are preserved by copying the input event. // - ID/Kind/Source/EmittedAt are preserved by copying the input event.
// - EffectiveAt is only overwritten when effectiveAt is non-zero. // - EffectiveAt is only overwritten when effectiveAt is non-zero.
// If effectiveAt is zero, any existing in.EffectiveAt is preserved. // If effectiveAt is zero, any existing in.EffectiveAt is preserved.
// - Payload floats are rounded to a stable wire-friendly precision (see round.go).
func Finalize(in event.Event, outSchema string, outPayload any, effectiveAt time.Time) (*event.Event, error) { func Finalize(in event.Event, outSchema string, outPayload any, effectiveAt time.Time) (*event.Event, error) {
out := in // Enforce stable numeric presentation for weather payloads before delegating to feedkit's
out.Schema = outSchema // generic envelope finalizer.
out.Payload = outPayload return fknormalize.FinalizeEvent(in, outSchema, RoundFloats(outPayload, DefaultFloatPrecision), effectiveAt)
if !effectiveAt.IsZero() {
t := effectiveAt.UTC()
out.EffectiveAt = &t
}
if err := out.Validate(); err != nil {
return nil, err
}
return &out, nil
} }

View File

@@ -0,0 +1,36 @@
package common
import (
"testing"
"time"
"gitea.maximumdirect.net/ejr/feedkit/event"
)
func TestFinalizeRoundsWeatherPayloadFloats(t *testing.T) {
type payload struct {
Value float64
}
in := event.Event{
ID: "evt-1",
Kind: event.Kind("observation"),
Source: "source-a",
EmittedAt: time.Date(2026, 3, 28, 12, 0, 0, 0, time.UTC),
Schema: "raw.example.v1",
Payload: map[string]any{"old": true},
}
out, err := Finalize(in, "weather.example.v1", payload{Value: 1.234567}, time.Time{})
if err != nil {
t.Fatalf("Finalize() unexpected error: %v", err)
}
got, ok := out.Payload.(payload)
if !ok {
t.Fatalf("Finalize() payload type = %T, want payload", out.Payload)
}
if got.Value != 1.2346 {
t.Fatalf("Finalize() rounded value = %v, want 1.2346", got.Value)
}
}

View File

@@ -0,0 +1,23 @@
// FILE: internal/normalizers/common/id.go
package common
import "fmt"
// SynthStationID formats a stable synthetic station identifier for providers that are
// coordinate-based rather than station-based.
//
// Example output:
//
// OPENMETEO(38.62700,-90.19940)
func SynthStationID(prefix string, lat, lon float64) string {
return fmt.Sprintf("%s(%.5f,%.5f)", prefix, lat, lon)
}
// SynthStationIDPtr is the pointer-friendly variant.
// If either coordinate is missing, it returns "" (unknown).
func SynthStationIDPtr(prefix string, lat, lon *float64) string {
if lat == nil || lon == nil {
return ""
}
return SynthStationID(prefix, *lat, *lon)
}

View File

@@ -2,11 +2,11 @@
package common package common
import ( import (
"encoding/json"
"fmt" "fmt"
"time" "time"
"gitea.maximumdirect.net/ejr/feedkit/event" "gitea.maximumdirect.net/ejr/feedkit/event"
fknormalize "gitea.maximumdirect.net/ejr/feedkit/processors/normalize"
) )
// DecodeJSONPayload extracts the event payload as bytes and unmarshals it into T. // DecodeJSONPayload extracts the event payload as bytes and unmarshals it into T.
@@ -18,19 +18,7 @@ import (
// Errors include a small amount of stage context ("extract payload", "decode raw payload"). // Errors include a small amount of stage context ("extract payload", "decode raw payload").
// Callers typically wrap these with a provider/kind label. // Callers typically wrap these with a provider/kind label.
func DecodeJSONPayload[T any](in event.Event) (T, error) { func DecodeJSONPayload[T any](in event.Event) (T, error) {
var zero T return fknormalize.DecodeJSONPayload[T](in)
b, err := PayloadBytes(in)
if err != nil {
return zero, fmt.Errorf("extract payload: %w", err)
}
var parsed T
if err := json.Unmarshal(b, &parsed); err != nil {
return zero, fmt.Errorf("decode raw payload: %w", err)
}
return parsed, nil
} }
// NormalizeJSON is a convenience wrapper for the common JSON-normalizer pattern: // NormalizeJSON is a convenience wrapper for the common JSON-normalizer pattern:
@@ -42,6 +30,10 @@ func DecodeJSONPayload[T any](in event.Event) (T, error) {
// label should be short and specific, e.g. "openweather observation". // label should be short and specific, e.g. "openweather observation".
// outSchema should be the canonical schema constant. // outSchema should be the canonical schema constant.
// build should contain ONLY provider/domain mapping logic. // build should contain ONLY provider/domain mapping logic.
//
// Error policy:
// - NormalizeJSON wraps ALL failures with consistent context: "<label> normalize: <stage>: ..."
// - build() should return specific errors without repeating the label prefix.
func NormalizeJSON[T any, P any]( func NormalizeJSON[T any, P any](
in event.Event, in event.Event,
label string, label string,
@@ -55,8 +47,7 @@ func NormalizeJSON[T any, P any](
payload, effectiveAt, err := build(parsed) payload, effectiveAt, err := build(parsed)
if err != nil { if err != nil {
// build() should already include provider-specific context where appropriate. return nil, fmt.Errorf("%s normalize: build: %w", label, err)
return nil, err
} }
out, err := Finalize(in, outSchema, payload, effectiveAt) out, err := Finalize(in, outSchema, payload, effectiveAt)

View File

@@ -1,53 +0,0 @@
package common
import (
"encoding/json"
"fmt"
"gitea.maximumdirect.net/ejr/feedkit/event"
)
// PayloadBytes extracts a JSON payload into bytes suitable for json.Unmarshal.
//
// Supported payload shapes (weatherfeeder convention):
// - json.RawMessage (recommended for raw events)
// - []byte
// - string (assumed to contain JSON)
// - map[string]any (re-marshaled to JSON)
//
// If you add other raw representations later, extend this function.
func PayloadBytes(e event.Event) ([]byte, error) {
if e.Payload == nil {
return nil, fmt.Errorf("payload is nil")
}
switch v := e.Payload.(type) {
case json.RawMessage:
if len(v) == 0 {
return nil, fmt.Errorf("payload is empty json.RawMessage")
}
return []byte(v), nil
case []byte:
if len(v) == 0 {
return nil, fmt.Errorf("payload is empty []byte")
}
return v, nil
case string:
if v == "" {
return nil, fmt.Errorf("payload is empty string")
}
return []byte(v), nil
case map[string]any:
b, err := json.Marshal(v)
if err != nil {
return nil, fmt.Errorf("marshal map payload: %w", err)
}
return b, nil
default:
return nil, fmt.Errorf("unsupported payload type %T", e.Payload)
}
}

View File

@@ -0,0 +1,212 @@
// FILE: ./internal/normalizers/common/round.go
package common
import (
"math"
"reflect"
)
// DefaultFloatPrecision is the project-wide wire-format policy for floating-point
// values in canonical payloads (weather.* schemas).
//
// Note: encoding/json will not necessarily print trailing zeros (e.g. 1.50 -> 1.5),
// but values will be *rounded* to this number of digits after the decimal point.
const DefaultFloatPrecision = 4
// RoundFloats returns a copy of v with all float32/float64 values (including pointers,
// slices, arrays, maps, and nested exported-struct fields) rounded to `decimals` digits
// after the decimal point.
//
// This is a best-effort helper meant for presentation stability. If reflection hits an
// unsupported/opaque type (e.g. structs with unexported fields like time.Time), that
// subtree is left unchanged.
func RoundFloats(v any, decimals int) any {
if v == nil || decimals < 0 {
return v
}
defer func() {
// Never let presentation formatting crash the pipeline.
_ = recover()
}()
rv := reflect.ValueOf(v)
out := roundValue(rv, decimals)
if !out.IsValid() {
return v
}
return out.Interface()
}
func roundValue(v reflect.Value, decimals int) reflect.Value {
if !v.IsValid() {
return v
}
// Unwrap interfaces.
if v.Kind() == reflect.Interface {
if v.IsNil() {
return v
}
elem := roundValue(v.Elem(), decimals)
// Re-wrap in the same interface type.
out := reflect.New(v.Type()).Elem()
if elem.IsValid() && elem.Type().AssignableTo(v.Type()) {
out.Set(elem)
return out
}
if elem.IsValid() && elem.Type().ConvertibleTo(v.Type()) {
out.Set(elem.Convert(v.Type()))
return out
}
// If we can't sensibly re-wrap, just keep the original.
return v
}
// Copy pointers (and round their targets).
if v.Kind() == reflect.Pointer {
if v.IsNil() {
return v
}
// If the pointed-to type is an opaque struct (e.g. time.Time), keep as-is.
if v.Elem().Kind() == reflect.Struct && isOpaqueStruct(v.Elem().Type()) {
return v
}
elem := roundValue(v.Elem(), decimals)
p := reflect.New(v.Type().Elem())
if elem.IsValid() && elem.Type().AssignableTo(v.Type().Elem()) {
p.Elem().Set(elem)
} else if elem.IsValid() && elem.Type().ConvertibleTo(v.Type().Elem()) {
p.Elem().Set(elem.Convert(v.Type().Elem()))
} else {
p.Elem().Set(v.Elem())
}
return p
}
switch v.Kind() {
case reflect.Float32, reflect.Float64:
f := v.Convert(reflect.TypeOf(float64(0))).Float()
r := roundFloat64(f, decimals)
return reflect.ValueOf(r).Convert(v.Type())
case reflect.Struct:
// Avoid reconstructing opaque structs (time.Time has unexported fields).
if isOpaqueStruct(v.Type()) {
return v
}
out := reflect.New(v.Type()).Elem()
out.Set(v) // start from a copy, then replace rounded fields
t := v.Type()
for i := 0; i < v.NumField(); i++ {
sf := t.Field(i)
// Only exported fields are safely settable across packages.
if sf.PkgPath != "" {
continue
}
fv := v.Field(i)
rf := roundValue(fv, decimals)
of := out.Field(i)
if !of.CanSet() {
continue
}
if rf.IsValid() && rf.Type().AssignableTo(of.Type()) {
of.Set(rf)
} else if rf.IsValid() && rf.Type().ConvertibleTo(of.Type()) {
of.Set(rf.Convert(of.Type()))
}
}
return out
case reflect.Slice:
if v.IsNil() {
return v
}
out := reflect.MakeSlice(v.Type(), v.Len(), v.Len())
for i := 0; i < v.Len(); i++ {
ev := v.Index(i)
re := roundValue(ev, decimals)
if re.IsValid() && re.Type().AssignableTo(out.Index(i).Type()) {
out.Index(i).Set(re)
} else if re.IsValid() && re.Type().ConvertibleTo(out.Index(i).Type()) {
out.Index(i).Set(re.Convert(out.Index(i).Type()))
} else {
out.Index(i).Set(ev)
}
}
return out
case reflect.Array:
out := reflect.New(v.Type()).Elem()
out.Set(v)
for i := 0; i < v.Len(); i++ {
ev := v.Index(i)
re := roundValue(ev, decimals)
if re.IsValid() && re.Type().AssignableTo(out.Index(i).Type()) {
out.Index(i).Set(re)
} else if re.IsValid() && re.Type().ConvertibleTo(out.Index(i).Type()) {
out.Index(i).Set(re.Convert(out.Index(i).Type()))
} else {
out.Index(i).Set(ev)
}
}
return out
case reflect.Map:
if v.IsNil() {
return v
}
out := reflect.MakeMapWithSize(v.Type(), v.Len())
iter := v.MapRange()
for iter.Next() {
k := iter.Key()
mv := iter.Value()
rv := roundValue(mv, decimals)
if rv.IsValid() && rv.Type().AssignableTo(v.Type().Elem()) {
out.SetMapIndex(k, rv)
} else if rv.IsValid() && rv.Type().ConvertibleTo(v.Type().Elem()) {
out.SetMapIndex(k, rv.Convert(v.Type().Elem()))
} else {
out.SetMapIndex(k, mv)
}
}
return out
default:
// ints, strings, bools, time.Time (handled as opaque), etc.
return v
}
}
func roundFloat64(f float64, decimals int) float64 {
if decimals <= 0 {
return math.Round(f)
}
pow := math.Pow10(decimals)
return math.Round(f*pow) / pow
}
// isOpaqueStruct returns true for structs that are unsafe/unhelpful to reconstruct via reflection.
// Any struct containing unexported fields (e.g. time.Time) is treated as opaque.
func isOpaqueStruct(t reflect.Type) bool {
if t.Kind() != reflect.Struct {
return false
}
for i := 0; i < t.NumField(); i++ {
if t.Field(i).PkgPath != "" {
return true
}
}
return false
}

View File

@@ -4,7 +4,7 @@ package common
import ( import (
"strings" "strings"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/model" "gitea.maximumdirect.net/ejr/weatherfeeder/model"
) )
// WMOFromTextDescription is a cross-provider fallback that tries to infer a WMO code // WMOFromTextDescription is a cross-provider fallback that tries to infer a WMO code

View File

@@ -8,7 +8,7 @@
// transforming provider-specific raw payloads into canonical internal models. // transforming provider-specific raw payloads into canonical internal models.
// //
// This package is domain code (weatherfeeder). feedkits normalize package is // This package is domain code (weatherfeeder). feedkits normalize package is
// infrastructure (registry + processor). // infrastructure (normalizer contracts + processor).
// //
// Directory layout (required) // Directory layout (required)
// --------------------------- // ---------------------------
@@ -29,19 +29,21 @@
// //
// 1. One normalizer per file. // 1. One normalizer per file.
// Each file contains exactly one Normalizer implementation (one type that // Each file contains exactly one Normalizer implementation (one type that
// satisfies feedkit/normalize.Normalizer). // satisfies feedkit/processors/normalize.Normalizer).
// Helper files are encouraged (types.go, common.go, mapping.go, etc.) as long // Helper files are encouraged (types.go, common.go, mapping.go, etc.) as long
// as they do not define additional Normalizer types. // as they do not define additional Normalizer types.
// //
// 2. Provider-level shared helpers live under the provider directory: // 2. Provider-level shared helpers live under the provider directory:
// internal/normalizers/<provider>/ // internal/providers/<provider>/
//
// Use this for provider-specific quirks that should be shared by BOTH sources
// and normalizers (time parsing, URL/unit invariants, ID normalization, etc.).
// Keep these helpers pure (no I/O) and easy to test.
// //
// You may use multiple helper files (recommended) when it improves clarity: // You may use multiple helper files (recommended) when it improves clarity:
// - types.go (provider JSON structs) // - types.go (provider JSON structs)
// - common.go (provider-shared helpers) // - common.go (provider-shared helpers)
// - mapping.go (provider mapping logic) // - mapping.go (provider mapping logic)
// Use common.go only when you truly have “shared across multiple normalizers
// within this provider” helpers.
// //
// 3. Cross-provider helpers live in: // 3. Cross-provider helpers live in:
// internal/normalizers/common/ // internal/normalizers/common/
@@ -134,21 +136,22 @@
// //
// Registration pattern // Registration pattern
// -------------------- // --------------------
// feedkit normalization uses a match-driven registry (first match wins). // feedkit normalization uses an ordered normalizer list ("first match wins").
// //
// Provider subpackages should expose: // Provider subpackages should expose:
// //
// func Register(reg *normalize.Registry) // func Register(in []normalize.Normalizer) []normalize.Normalizer
// //
// And internal/normalizers/builtins.go should provide one entrypoint: // And internal/normalizers/builtins.go should provide one entrypoint:
// //
// func RegisterBuiltins(reg *normalize.Registry) // func RegisterBuiltins(in []normalize.Normalizer) []normalize.Normalizer
// //
// which calls each providers Register() in a stable order. // which appends each provider's normalizers in a stable order and is then passed
// to normalize.NewProcessor(...).
// //
// Registry ordering // Normalizer ordering
// ----------------------------- // -----------------------------
// feedkit normalization uses a match-driven registry (“first match wins”). // feedkit normalization is "first match wins" by list order.
// Therefore order matters: // Therefore order matters:
// //
// - Register more specific normalizers before more general ones. // - Register more specific normalizers before more general ones.

View File

@@ -0,0 +1,238 @@
// FILE: internal/normalizers/nws/alerts.go
package nws
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
"gitea.maximumdirect.net/ejr/feedkit/event"
normcommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/common"
nwscommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/nws"
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
// AlertsNormalizer converts:
//
// standards.SchemaRawNWSAlertsV1 -> standards.SchemaWeatherAlertV1
//
// It interprets NWS /alerts FeatureCollection payloads (GeoJSON-ish) and maps them into
// the canonical model.WeatherAlertRun representation.
//
// Caveats / policy:
// 1. Run.AsOf prefers the collection-level "updated" timestamp. If missing/unparseable,
// we fall back to the latest per-alert timestamp, and then to the input events
// EffectiveAt/EmittedAt.
// 2. Alert timing fields are best-effort parsed; invalid timestamps do not fail the
// entire normalization (they are left nil).
// 3. Some fields are intentionally passed through as strings (severity/urgency/etc.)
// since canonical vocabularies may evolve later.
type AlertsNormalizer struct{}
func (AlertsNormalizer) Match(e event.Event) bool {
return strings.TrimSpace(e.Schema) == standards.SchemaRawNWSAlertsV1
}
func (AlertsNormalizer) Normalize(ctx context.Context, in event.Event) (*event.Event, error) {
_ = ctx // normalization is pure/CPU; keep ctx for future expensive steps
// If we can't derive AsOf from the payload, fall back to the existing event envelope.
fallbackAsOf := in.EmittedAt.UTC()
if in.EffectiveAt != nil && !in.EffectiveAt.IsZero() {
fallbackAsOf = in.EffectiveAt.UTC()
}
return normcommon.NormalizeJSON(
in,
"nws alerts",
standards.SchemaWeatherAlertV1,
func(parsed nwsAlertsResponse) (model.WeatherAlertRun, time.Time, error) {
return buildAlerts(parsed, fallbackAsOf)
},
)
}
// buildAlerts contains the domain mapping logic (provider -> canonical model).
func buildAlerts(parsed nwsAlertsResponse, fallbackAsOf time.Time) (model.WeatherAlertRun, time.Time, error) {
// 1) Determine AsOf (required by canonical model; also used as EffectiveAt).
asOf := nwscommon.ParseTimeBestEffort(parsed.Updated)
if asOf.IsZero() {
asOf = latestAlertTimestamp(parsed.Features)
}
if asOf.IsZero() {
asOf = fallbackAsOf.UTC()
}
run := model.WeatherAlertRun{
LocationID: "",
LocationName: strings.TrimSpace(parsed.Title),
AsOf: asOf,
Latitude: nil,
Longitude: nil,
Alerts: nil,
}
// 2) Map each feature into a canonical WeatherAlert.
alerts := make([]model.WeatherAlert, 0, len(parsed.Features))
for i, f := range parsed.Features {
p := f.Properties
id := firstNonEmpty(strings.TrimSpace(f.ID), strings.TrimSpace(p.ID), strings.TrimSpace(p.Identifier))
if id == "" {
// NWS usually supplies an ID, but be defensive. Prefer a stable-ish synth ID.
// Include the run as-of time to reduce collisions across snapshots.
id = fmt.Sprintf("nws:alert:%s:%d", asOf.UTC().Format(time.RFC3339Nano), i)
}
sent := nwscommon.ParseTimePtr(p.Sent)
effective := nwscommon.ParseTimePtr(p.Effective)
onset := nwscommon.ParseTimePtr(p.Onset)
// Expires: prefer "expires"; fall back to "ends" if present.
expires := nwscommon.ParseTimePtr(p.Expires)
if expires == nil {
expires = nwscommon.ParseTimePtr(p.Ends)
}
refs := parseNWSAlertReferences(p.References)
alert := model.WeatherAlert{
ID: id,
Event: strings.TrimSpace(p.Event),
Headline: strings.TrimSpace(p.Headline),
Severity: strings.TrimSpace(p.Severity),
Urgency: strings.TrimSpace(p.Urgency),
Certainty: strings.TrimSpace(p.Certainty),
Status: strings.TrimSpace(p.Status),
MessageType: strings.TrimSpace(p.MessageType),
Category: strings.TrimSpace(p.Category),
Response: strings.TrimSpace(p.Response),
Description: strings.TrimSpace(p.Description),
Instruction: strings.TrimSpace(p.Instruction),
Sent: sent,
Effective: effective,
Onset: onset,
Expires: expires,
AreaDescription: strings.TrimSpace(p.AreaDesc),
SenderName: firstNonEmpty(strings.TrimSpace(p.SenderName), strings.TrimSpace(p.Sender)),
References: refs,
}
// Headline fallback (NWS commonly provides it, but not guaranteed).
if alert.Headline == "" {
alert.Headline = alert.Event
}
alerts = append(alerts, alert)
}
run.Alerts = alerts
// EffectiveAt policy for alerts: treat AsOf as the effective time (dedupe-friendly).
return run, asOf, nil
}
// latestAlertTimestamp scans alert features for the most recent timestamp.
// It prefers Sent/Effective, and falls back to Expires/Ends when needed.
func latestAlertTimestamp(features []nwsAlertFeature) time.Time {
var latest time.Time
for _, f := range features {
p := f.Properties
candidates := []string{
p.Sent,
p.Effective,
p.Expires,
p.Ends,
p.Onset,
}
for _, s := range candidates {
t := nwscommon.ParseTimeBestEffort(s)
if t.IsZero() {
continue
}
if latest.IsZero() || t.After(latest) {
latest = t
}
}
}
return latest
}
func firstNonEmpty(vals ...string) string {
for _, v := range vals {
if strings.TrimSpace(v) != "" {
return strings.TrimSpace(v)
}
}
return ""
}
// parseNWSAlertReferences tries to interpret the NWS "references" field, which may
// vary by endpoint/version. We accept the common object-array form and a few
// degraded shapes (string array / single string).
func parseNWSAlertReferences(raw json.RawMessage) []model.AlertReference {
if len(raw) == 0 {
return nil
}
// Most common: array of objects.
var objs []nwsAlertReference
if err := json.Unmarshal(raw, &objs); err == nil && len(objs) > 0 {
out := make([]model.AlertReference, 0, len(objs))
for _, r := range objs {
ref := model.AlertReference{
ID: strings.TrimSpace(r.ID),
Identifier: strings.TrimSpace(r.Identifier),
Sender: strings.TrimSpace(r.Sender),
Sent: nwscommon.ParseTimePtr(r.Sent),
}
// If only Identifier is present, preserve it as ID too (useful downstream).
if ref.ID == "" && ref.Identifier != "" {
ref.ID = ref.Identifier
}
out = append(out, ref)
}
return out
}
// Sometimes: array of strings.
var strs []string
if err := json.Unmarshal(raw, &strs); err == nil && len(strs) > 0 {
out := make([]model.AlertReference, 0, len(strs))
for _, s := range strs {
id := strings.TrimSpace(s)
if id == "" {
continue
}
out = append(out, model.AlertReference{ID: id})
}
if len(out) > 0 {
return out
}
}
// Rare: single string.
var single string
if err := json.Unmarshal(raw, &single); err == nil {
id := strings.TrimSpace(single)
if id != "" {
return []model.AlertReference{{ID: id}}
}
}
return nil
}

View File

@@ -0,0 +1,74 @@
// FILE: internal/normalizers/nws/day_night.go
package nws
import (
"math"
"time"
)
const (
degToRad = math.Pi / 180.0
radToDeg = 180.0 / math.Pi
)
func observationLatLon(coords []float64) (lat *float64, lon *float64) {
if len(coords) < 2 {
return nil, nil
}
latVal := coords[1]
lonVal := coords[0]
return &latVal, &lonVal
}
// isDayFromLatLonTime estimates day/night from solar elevation.
// Uses a refraction-adjusted cutoff (-0.833 degrees).
func isDayFromLatLonTime(lat, lon float64, ts time.Time) *bool {
if ts.IsZero() || math.IsNaN(lat) || math.IsNaN(lon) || math.IsInf(lat, 0) || math.IsInf(lon, 0) {
return nil
}
if lat < -90.0 || lat > 90.0 || lon < -180.0 || lon > 180.0 {
return nil
}
t := ts.UTC()
day := float64(t.YearDay())
hour := float64(t.Hour())
min := float64(t.Minute())
sec := float64(t.Second()) + float64(t.Nanosecond())/1e9
utcHours := hour + min/60.0 + sec/3600.0
gamma := 2.0 * math.Pi / 365.0 * (day - 1.0 + (utcHours-12.0)/24.0)
eqtime := 229.18 * (0.000075 + 0.001868*math.Cos(gamma) - 0.032077*math.Sin(gamma) -
0.014615*math.Cos(2.0*gamma) - 0.040849*math.Sin(2.0*gamma))
decl := 0.006918 - 0.399912*math.Cos(gamma) + 0.070257*math.Sin(gamma) -
0.006758*math.Cos(2.0*gamma) + 0.000907*math.Sin(2.0*gamma) -
0.002697*math.Cos(3.0*gamma) + 0.00148*math.Sin(3.0*gamma)
timeOffset := eqtime + 4.0*lon
trueSolarMinutes := hour*60.0 + min + sec/60.0 + timeOffset
for trueSolarMinutes < 0 {
trueSolarMinutes += 1440.0
}
for trueSolarMinutes >= 1440.0 {
trueSolarMinutes -= 1440.0
}
hourAngleDeg := trueSolarMinutes/4.0 - 180.0
ha := hourAngleDeg * degToRad
latRad := lat * degToRad
cosZenith := math.Sin(latRad)*math.Sin(decl) + math.Cos(latRad)*math.Cos(decl)*math.Cos(ha)
if cosZenith > 1.0 {
cosZenith = 1.0
} else if cosZenith < -1.0 {
cosZenith = -1.0
}
zenith := math.Acos(cosZenith)
elevation := 90.0 - zenith*radToDeg
isDay := elevation > -0.833
return &isDay
}

View File

@@ -0,0 +1,294 @@
// FILE: internal/normalizers/nws/forecast.go
package nws
import (
"context"
"fmt"
"strings"
"time"
"gitea.maximumdirect.net/ejr/feedkit/event"
normcommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/common"
nwscommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/nws"
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
// ForecastNormalizer converts:
//
// standards.SchemaRawNWSHourlyForecastV1 -> standards.SchemaWeatherForecastV1
// standards.SchemaRawNWSNarrativeForecastV1 -> standards.SchemaWeatherForecastV1
//
// It keeps one NWS forecast normalization entrypoint and dispatches to product-specific
// builders by raw schema.
//
// Caveats / policy:
// 1. NWS forecast periods do not include METAR presentWeather phenomena, so ConditionCode
// is inferred from period.shortForecast (with a conservative icon-based fallback).
// 2. Temperature is converted to °C when NWS supplies °F.
// 3. WindSpeed is parsed from strings like "9 mph" / "10 to 15 mph" and converted to km/h.
type ForecastNormalizer struct{}
func (ForecastNormalizer) Match(e event.Event) bool {
switch strings.TrimSpace(e.Schema) {
case standards.SchemaRawNWSHourlyForecastV1:
return true
case standards.SchemaRawNWSNarrativeForecastV1:
return true
default:
return false
}
}
func (ForecastNormalizer) Normalize(ctx context.Context, in event.Event) (*event.Event, error) {
_ = ctx // normalization is pure/CPU; keep ctx for future expensive steps
return normalizeForecastEventBySchema(in)
}
func normalizeForecastEventBySchema(in event.Event) (*event.Event, error) {
switch strings.TrimSpace(in.Schema) {
case standards.SchemaRawNWSHourlyForecastV1:
return normalizeHourlyForecastEvent(in)
case standards.SchemaRawNWSNarrativeForecastV1:
return normalizeNarrativeForecastEvent(in)
default:
return nil, fmt.Errorf("unsupported nws forecast schema %q", strings.TrimSpace(in.Schema))
}
}
func normalizeHourlyForecastEvent(in event.Event) (*event.Event, error) {
return normcommon.NormalizeJSON(
in,
"nws hourly forecast",
standards.SchemaWeatherForecastV1,
buildHourlyForecast,
)
}
func normalizeNarrativeForecastEvent(in event.Event) (*event.Event, error) {
return normcommon.NormalizeJSON(
in,
"nws narrative forecast",
standards.SchemaWeatherForecastV1,
buildNarrativeForecast,
)
}
type forecastPeriodMapper[T any] func(idx int, period T) (model.WeatherForecastPeriod, error)
// buildHourlyForecast contains hourly forecast mapping logic (provider -> canonical model).
func buildHourlyForecast(parsed nwsHourlyForecastResponse) (model.WeatherForecastRun, time.Time, error) {
return buildForecastRun(
parsed.Properties.GeneratedAt,
parsed.Properties.UpdateTime,
parsed.Geometry.Coordinates,
parsed.Properties.Elevation.Value,
model.ForecastProductHourly,
parsed.Properties.Periods,
mapHourlyForecastPeriod,
)
}
// buildNarrativeForecast contains narrative forecast mapping logic (provider -> canonical model).
func buildNarrativeForecast(parsed nwsNarrativeForecastResponse) (model.WeatherForecastRun, time.Time, error) {
return buildForecastRun(
parsed.Properties.GeneratedAt,
parsed.Properties.UpdateTime,
parsed.Geometry.Coordinates,
parsed.Properties.Elevation.Value,
model.ForecastProductNarrative,
parsed.Properties.Periods,
mapNarrativeForecastPeriod,
)
}
func buildForecastRun[T any](
generatedAt string,
updateTime string,
coordinates [][][]float64,
elevation *float64,
product model.ForecastProduct,
srcPeriods []T,
mapPeriod forecastPeriodMapper[T],
) (model.WeatherForecastRun, time.Time, error) {
issuedAt, updatedAt, err := parseForecastRunTimes(generatedAt, updateTime)
if err != nil {
return model.WeatherForecastRun{}, time.Time{}, err
}
// Best-effort location centroid from the GeoJSON polygon (optional).
lat, lon := centroidLatLon(coordinates)
run := newForecastRunBase(
issuedAt,
updatedAt,
product,
lat,
lon,
elevation,
)
periods := make([]model.WeatherForecastPeriod, 0, len(srcPeriods))
for i, p := range srcPeriods {
period, err := mapPeriod(i, p)
if err != nil {
return model.WeatherForecastRun{}, time.Time{}, err
}
periods = append(periods, period)
}
run.Periods = periods
// EffectiveAt policy for forecasts: treat IssuedAt as the effective time (dedupe-friendly).
return run, issuedAt, nil
}
func parseForecastRunTimes(generatedAt, updateTime string) (time.Time, *time.Time, error) {
issuedStr := strings.TrimSpace(generatedAt)
if issuedStr == "" {
return time.Time{}, nil, fmt.Errorf("missing properties.generatedAt")
}
issuedAt, err := nwscommon.ParseTime(issuedStr)
if err != nil {
return time.Time{}, nil, fmt.Errorf("invalid properties.generatedAt %q: %w", issuedStr, err)
}
issuedAt = issuedAt.UTC()
var updatedAt *time.Time
if s := strings.TrimSpace(updateTime); s != "" {
if t, err := nwscommon.ParseTime(s); err == nil {
tt := t.UTC()
updatedAt = &tt
}
}
return issuedAt, updatedAt, nil
}
func newForecastRunBase(
issuedAt time.Time,
updatedAt *time.Time,
product model.ForecastProduct,
lat, lon, elevation *float64,
) model.WeatherForecastRun {
return model.WeatherForecastRun{
LocationID: "",
LocationName: "",
IssuedAt: issuedAt,
UpdatedAt: updatedAt,
Product: product,
Latitude: lat,
Longitude: lon,
ElevationMeters: elevation,
Periods: nil,
}
}
func parseForecastPeriodWindow(startStr, endStr string, idx int) (time.Time, time.Time, error) {
startStr = strings.TrimSpace(startStr)
endStr = strings.TrimSpace(endStr)
if startStr == "" || endStr == "" {
return time.Time{}, time.Time{}, fmt.Errorf("periods[%d]: missing startTime/endTime", idx)
}
start, err := nwscommon.ParseTime(startStr)
if err != nil {
return time.Time{}, time.Time{}, fmt.Errorf("periods[%d].startTime invalid %q: %w", idx, startStr, err)
}
end, err := nwscommon.ParseTime(endStr)
if err != nil {
return time.Time{}, time.Time{}, fmt.Errorf("periods[%d].endTime invalid %q: %w", idx, endStr, err)
}
return start.UTC(), end.UTC(), nil
}
func mapHourlyForecastPeriod(idx int, p nwsHourlyForecastPeriod) (model.WeatherForecastPeriod, error) {
start, end, err := parseForecastPeriodWindow(p.StartTime, p.EndTime, idx)
if err != nil {
return model.WeatherForecastPeriod{}, err
}
// NWS hourly supplies isDaytime; make it a pointer to match the canonical model.
var isDay *bool
if p.IsDaytime != nil {
b := *p.IsDaytime
isDay = &b
}
tempC := tempCFromNWS(p.Temperature, p.TemperatureUnit)
// Infer WMO from shortForecast (and fall back to icon token).
providerDesc := strings.TrimSpace(p.ShortForecast)
wmo := wmoFromNWSForecast(providerDesc, p.Icon, tempC)
return model.WeatherForecastPeriod{
StartTime: start,
EndTime: end,
Name: strings.TrimSpace(p.Name),
IsDay: isDay,
ConditionCode: wmo,
// For forecasts, keep provider short forecast text as the human-facing description.
TextDescription: providerDesc,
TemperatureC: tempC,
DewpointC: p.Dewpoint.Value,
RelativeHumidityPercent: p.RelativeHumidity.Value,
WindDirectionDegrees: parseNWSWindDirectionDegrees(p.WindDirection),
WindSpeedKmh: parseNWSWindSpeedKmh(p.WindSpeed),
ProbabilityOfPrecipitationPercent: p.ProbabilityOfPrecipitation.Value,
}, nil
}
func mapNarrativeForecastPeriod(idx int, p nwsNarrativeForecastPeriod) (model.WeatherForecastPeriod, error) {
start, end, err := parseForecastPeriodWindow(p.StartTime, p.EndTime, idx)
if err != nil {
return model.WeatherForecastPeriod{}, err
}
// NWS narrative supplies isDaytime; make it a pointer to match the canonical model.
var isDay *bool
if p.IsDaytime != nil {
b := *p.IsDaytime
isDay = &b
}
tempC := tempCFromNWS(p.Temperature, p.TemperatureUnit)
// Infer WMO from shortForecast (and fall back to icon token).
shortForecast := strings.TrimSpace(p.ShortForecast)
wmo := wmoFromNWSForecast(shortForecast, p.Icon, tempC)
textDescription := strings.TrimSpace(p.DetailedForecast)
if textDescription == "" {
textDescription = shortForecast
}
return model.WeatherForecastPeriod{
StartTime: start,
EndTime: end,
Name: strings.TrimSpace(p.Name),
IsDay: isDay,
ConditionCode: wmo,
TextDescription: textDescription,
TemperatureC: tempC,
WindDirectionDegrees: parseNWSWindDirectionDegrees(p.WindDirection),
WindSpeedKmh: parseNWSWindSpeedKmh(p.WindSpeed),
ProbabilityOfPrecipitationPercent: p.ProbabilityOfPrecipitation.Value,
}, nil
}

View File

@@ -0,0 +1,72 @@
package nws
import (
"context"
"fmt"
"strings"
"gitea.maximumdirect.net/ejr/feedkit/event"
normcommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/common"
nwscommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/nws"
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
type ForecastDiscussionNormalizer struct{}
func (ForecastDiscussionNormalizer) Match(e event.Event) bool {
return strings.TrimSpace(e.Schema) == standards.SchemaRawNWSForecastDiscussionV1
}
func (ForecastDiscussionNormalizer) Normalize(ctx context.Context, in event.Event) (*event.Event, error) {
_ = ctx
rawHTML, err := decodeStringPayload(in.Payload)
if err != nil {
return nil, fmt.Errorf("nws forecast discussion normalize: %w", err)
}
parsed, err := nwscommon.ParseForecastDiscussionHTML(rawHTML)
if err != nil {
return nil, fmt.Errorf("nws forecast discussion normalize: build: %w", err)
}
payload := model.WeatherForecastDiscussion{
OfficeID: strings.TrimSpace(parsed.OfficeID),
OfficeName: strings.TrimSpace(parsed.OfficeName),
Product: model.ForecastDiscussionProduct(strings.TrimSpace(parsed.Product)),
IssuedAt: parsed.IssuedAt.UTC(),
UpdatedAt: parsed.UpdatedAt,
KeyMessages: append([]string(nil), parsed.KeyMessages...),
ShortTerm: mapForecastDiscussionSection(parsed.ShortTerm),
LongTerm: mapForecastDiscussionSection(parsed.LongTerm),
}
out, err := normcommon.Finalize(in, standards.SchemaWeatherForecastDiscussionV1, payload, payload.IssuedAt)
if err != nil {
return nil, fmt.Errorf("nws forecast discussion normalize: %w", err)
}
return out, nil
}
func mapForecastDiscussionSection(in *nwscommon.ForecastDiscussionSection) *model.WeatherForecastDiscussionSection {
if in == nil {
return nil
}
return &model.WeatherForecastDiscussionSection{
Qualifier: strings.TrimSpace(in.Qualifier),
IssuedAt: in.IssuedAt,
Text: strings.TrimSpace(in.Text),
}
}
func decodeStringPayload(payload any) (string, error) {
switch v := payload.(type) {
case string:
return v, nil
case []byte:
return string(v), nil
default:
return "", fmt.Errorf("extract payload: expected string payload, got %T", payload)
}
}

View File

@@ -0,0 +1,130 @@
package nws
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
func TestForecastDiscussionNormalizerProducesCanonicalSchema(t *testing.T) {
rawHTML := loadForecastDiscussionSampleHTML(t)
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
ID: "evt-discussion-1",
Kind: event.Kind("forecast_discussion"),
Source: "nws-discussion-test",
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
Schema: standards.SchemaRawNWSForecastDiscussionV1,
Payload: rawHTML,
})
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
if out == nil {
t.Fatalf("Normalize() returned nil output")
}
if out.Schema != standards.SchemaWeatherForecastDiscussionV1 {
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherForecastDiscussionV1)
}
if out.Kind != event.Kind("forecast_discussion") {
t.Fatalf("Kind = %q, want forecast_discussion", out.Kind)
}
payload, ok := out.Payload.(model.WeatherForecastDiscussion)
if !ok {
t.Fatalf("Payload type = %T, want model.WeatherForecastDiscussion", out.Payload)
}
if payload.OfficeID != "LSX" {
t.Fatalf("OfficeID = %q, want LSX", payload.OfficeID)
}
if payload.Product != model.ForecastDiscussionProductAFD {
t.Fatalf("Product = %q, want %q", payload.Product, model.ForecastDiscussionProductAFD)
}
if len(payload.KeyMessages) != 3 {
t.Fatalf("KeyMessages len = %d, want 3", len(payload.KeyMessages))
}
if payload.ShortTerm == nil || payload.LongTerm == nil {
t.Fatalf("ShortTerm=%v LongTerm=%v, want both populated", payload.ShortTerm, payload.LongTerm)
}
if payload.ShortTerm.Qualifier != "(Through Late Sunday Night)" {
t.Fatalf("ShortTerm.Qualifier = %q", payload.ShortTerm.Qualifier)
}
if !strings.Contains(payload.ShortTerm.Text, "After a chilly morning") {
t.Fatalf("ShortTerm.Text = %q, want normalized prose", payload.ShortTerm.Text)
}
if strings.Contains(payload.ShortTerm.Text, "BRC") {
t.Fatalf("ShortTerm.Text contains signature: %q", payload.ShortTerm.Text)
}
if strings.Contains(payload.LongTerm.Text, "AVIATION") || strings.Contains(payload.LongTerm.Text, "WATCHES/WARNINGS/ADVISORIES") {
t.Fatalf("LongTerm.Text includes downstream sections: %q", payload.LongTerm.Text)
}
wantEffectiveAt := time.Date(2026, 3, 28, 19, 24, 0, 0, time.UTC)
if out.EffectiveAt == nil || !out.EffectiveAt.Equal(wantEffectiveAt) {
t.Fatalf("EffectiveAt = %v, want %s", out.EffectiveAt, wantEffectiveAt.Format(time.RFC3339))
}
}
func TestForecastDiscussionNormalizerRejectsMissingIssueTime(t *testing.T) {
_, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
ID: "evt-discussion-bad",
Kind: event.Kind("forecast_discussion"),
Source: "nws-discussion-test",
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
Schema: standards.SchemaRawNWSForecastDiscussionV1,
Payload: "<html><body><pre class=\"glossaryProduct\">National Weather Service Saint Louis MO</pre></body></html>",
})
if err == nil {
t.Fatalf("Normalize() error = nil, want error")
}
if !strings.Contains(err.Error(), "issue time") {
t.Fatalf("error = %q, want issue time context", err)
}
}
func TestForecastDiscussionNormalizerWireShapeHasNoUnexpectedKeys(t *testing.T) {
rawHTML := loadForecastDiscussionSampleHTML(t)
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
ID: "evt-discussion-2",
Kind: event.Kind("forecast_discussion"),
Source: "nws-discussion-test",
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
Schema: standards.SchemaRawNWSForecastDiscussionV1,
Payload: rawHTML,
})
if err != nil {
t.Fatalf("Normalize() error = %v", err)
}
b, err := json.Marshal(out.Payload)
if err != nil {
t.Fatalf("json.Marshal(payload) error = %v", err)
}
var got map[string]any
if err := json.Unmarshal(b, &got); err != nil {
t.Fatalf("json.Unmarshal(payload) error = %v", err)
}
for _, key := range []string{"sections", "aviation"} {
if _, ok := got[key]; ok {
t.Fatalf("unexpected key %q in canonical payload", key)
}
}
}
func loadForecastDiscussionSampleHTML(t *testing.T) string {
t.Helper()
path := filepath.Join("..", "..", "providers", "nws", "testdata", "forecast_discussion_sample.html")
b, err := os.ReadFile(path)
if err != nil {
t.Fatalf("os.ReadFile(%q) error = %v", path, err)
}
return string(b)
}

View File

@@ -0,0 +1,324 @@
package nws
import (
"encoding/json"
"math"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
func TestBuildHourlyForecastUsesShortForecastAsTextDescription(t *testing.T) {
parsed := nwsHourlyForecastResponse{}
parsed.Properties.GeneratedAt = "2026-03-16T18:00:00Z"
parsed.Properties.Periods = []nwsHourlyForecastPeriod{
{
StartTime: "2026-03-16T19:00:00Z",
EndTime: "2026-03-16T20:00:00Z",
ShortForecast: " Mostly Cloudy ",
DetailedForecast: "Clouds increasing overnight.",
Icon: "https://example.invalid/icon",
},
}
run, effectiveAt, err := buildHourlyForecast(parsed)
if err != nil {
t.Fatalf("buildHourlyForecast() error = %v", err)
}
if len(run.Periods) != 1 {
t.Fatalf("periods len = %d, want 1", len(run.Periods))
}
if got, want := run.Periods[0].TextDescription, "Mostly Cloudy"; got != want {
t.Fatalf("TextDescription = %q, want %q", got, want)
}
wantIssued := time.Date(2026, 3, 16, 18, 0, 0, 0, time.UTC)
if !run.IssuedAt.Equal(wantIssued) {
t.Fatalf("IssuedAt = %s, want %s", run.IssuedAt.Format(time.RFC3339), wantIssued.Format(time.RFC3339))
}
if !effectiveAt.Equal(wantIssued) {
t.Fatalf("effectiveAt = %s, want %s", effectiveAt.Format(time.RFC3339), wantIssued.Format(time.RFC3339))
}
assertNoLegacyForecastDescriptionKeys(t, run.Periods[0])
}
func TestBuildHourlyForecastPreservesUpdatedAtCentroidAndElevation(t *testing.T) {
parsed := nwsHourlyForecastResponse{}
parsed.Properties.GeneratedAt = "2026-03-16T18:00:00Z"
parsed.Properties.UpdateTime = "2026-03-16T18:30:00Z"
elevation := 123.4
parsed.Properties.Elevation.Value = &elevation
parsed.Geometry.Coordinates = [][][]float64{
{
{-90.0, 38.0},
{-89.0, 38.0},
{-89.0, 39.0},
{-90.0, 39.0},
},
}
parsed.Properties.Periods = []nwsHourlyForecastPeriod{
{
StartTime: "2026-03-16T19:00:00Z",
EndTime: "2026-03-16T20:00:00Z",
ShortForecast: "Cloudy",
},
}
run, effectiveAt, err := buildHourlyForecast(parsed)
if err != nil {
t.Fatalf("buildHourlyForecast() error = %v", err)
}
wantIssued := time.Date(2026, 3, 16, 18, 0, 0, 0, time.UTC)
wantUpdated := time.Date(2026, 3, 16, 18, 30, 0, 0, time.UTC)
if !run.IssuedAt.Equal(wantIssued) {
t.Fatalf("IssuedAt = %s, want %s", run.IssuedAt.Format(time.RFC3339), wantIssued.Format(time.RFC3339))
}
if run.UpdatedAt == nil || !run.UpdatedAt.Equal(wantUpdated) {
t.Fatalf("UpdatedAt = %v, want %s", run.UpdatedAt, wantUpdated.Format(time.RFC3339))
}
if run.Latitude == nil || math.Abs(*run.Latitude-38.5) > 0.0001 {
t.Fatalf("Latitude = %v, want 38.5", run.Latitude)
}
if run.Longitude == nil || math.Abs(*run.Longitude+89.5) > 0.0001 {
t.Fatalf("Longitude = %v, want -89.5", run.Longitude)
}
if run.ElevationMeters == nil || math.Abs(*run.ElevationMeters-elevation) > 0.0001 {
t.Fatalf("ElevationMeters = %v, want %.1f", run.ElevationMeters, elevation)
}
if !effectiveAt.Equal(wantIssued) {
t.Fatalf("effectiveAt = %s, want %s", effectiveAt.Format(time.RFC3339), wantIssued.Format(time.RFC3339))
}
}
func TestNormalizeForecastEventBySchemaRejectsUnsupportedSchema(t *testing.T) {
_, err := normalizeForecastEventBySchema(event.Event{
Schema: "raw.nws.daily.forecast.v1",
})
if err == nil {
t.Fatalf("normalizeForecastEventBySchema() expected unsupported schema error")
}
if !strings.Contains(err.Error(), "unsupported nws forecast schema") {
t.Fatalf("error = %q, want unsupported schema context", err)
}
}
func TestNormalizeForecastEventBySchemaRoutesHourly(t *testing.T) {
_, err := normalizeForecastEventBySchema(event.Event{
Schema: standards.SchemaRawNWSHourlyForecastV1,
Payload: map[string]any{"properties": map[string]any{}},
})
if err == nil {
t.Fatalf("normalizeForecastEventBySchema() expected build error for missing generatedAt")
}
if !strings.Contains(err.Error(), "missing properties.generatedAt") {
t.Fatalf("error = %q, want missing properties.generatedAt", err)
}
}
func TestNormalizeForecastEventBySchemaRoutesNarrative(t *testing.T) {
_, err := normalizeForecastEventBySchema(event.Event{
Schema: standards.SchemaRawNWSNarrativeForecastV1,
Payload: map[string]any{"properties": map[string]any{}},
})
if err == nil {
t.Fatalf("normalizeForecastEventBySchema() expected build error for missing generatedAt")
}
if !strings.Contains(err.Error(), "missing properties.generatedAt") {
t.Fatalf("error = %q, want missing properties.generatedAt", err)
}
}
func TestNormalizeForecastEventBySchemaProducesCanonicalWeatherForecastSchema(t *testing.T) {
tests := []struct {
name string
schema string
payload map[string]any
}{
{
name: "hourly",
schema: standards.SchemaRawNWSHourlyForecastV1,
payload: map[string]any{
"properties": map[string]any{
"generatedAt": "2026-03-16T18:00:00Z",
"periods": []map[string]any{
{
"startTime": "2026-03-16T19:00:00Z",
"endTime": "2026-03-16T20:00:00Z",
"shortForecast": "Cloudy",
},
},
},
},
},
{
name: "narrative",
schema: standards.SchemaRawNWSNarrativeForecastV1,
payload: map[string]any{
"properties": map[string]any{
"generatedAt": "2026-03-16T18:00:00Z",
"periods": []map[string]any{
{
"startTime": "2026-03-16T19:00:00Z",
"endTime": "2026-03-16T20:00:00Z",
"shortForecast": "Cloudy",
"detailedForecast": "Cloudy",
},
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
out, err := normalizeForecastEventBySchema(event.Event{
ID: "evt-1",
Kind: event.Kind("forecast"),
Source: "nws-test",
EmittedAt: time.Date(2026, 3, 16, 18, 0, 0, 0, time.UTC),
Schema: tt.schema,
Payload: tt.payload,
})
if err != nil {
t.Fatalf("normalizeForecastEventBySchema() error = %v", err)
}
if out == nil {
t.Fatalf("normalizeForecastEventBySchema() returned nil output")
}
if out.Schema != standards.SchemaWeatherForecastV1 {
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherForecastV1)
}
})
}
}
func TestBuildNarrativeForecastMapsExpectedFields(t *testing.T) {
parsed := nwsNarrativeForecastResponse{}
parsed.Properties.GeneratedAt = "2026-03-27T15:17:01Z"
isDay := true
tempF := 53.0
pop := 20.0
parsed.Properties.Periods = []nwsNarrativeForecastPeriod{
{
Name: "Today",
StartTime: "2026-03-27T10:00:00-05:00",
EndTime: "2026-03-27T18:00:00-05:00",
IsDaytime: &isDay,
Temperature: &tempF,
TemperatureUnit: "F",
WindSpeed: "10 to 14 mph",
WindDirection: "SW",
ShortForecast: "Partly Sunny",
DetailedForecast: " Partly sunny, with a high near 53. ",
ProbabilityOfPrecipitation: struct {
UnitCode string `json:"unitCode"`
Value *float64 `json:"value"`
}{
UnitCode: "wmoUnit:percent",
Value: &pop,
},
Icon: "https://api.weather.gov/icons/land/day/bkn?size=medium",
},
}
run, effectiveAt, err := buildNarrativeForecast(parsed)
if err != nil {
t.Fatalf("buildNarrativeForecast() error = %v", err)
}
if got, want := run.Product, model.ForecastProductNarrative; got != want {
t.Fatalf("Product = %q, want %q", got, want)
}
if len(run.Periods) != 1 {
t.Fatalf("periods len = %d, want 1", len(run.Periods))
}
p := run.Periods[0]
if got, want := p.TextDescription, "Partly sunny, with a high near 53."; got != want {
t.Fatalf("TextDescription = %q, want %q", got, want)
}
if p.TemperatureC == nil {
t.Fatalf("TemperatureC is nil, want converted value")
}
if math.Abs(*p.TemperatureC-11.6666666667) > 0.0001 {
t.Fatalf("TemperatureC = %.6f, want ~11.6667", *p.TemperatureC)
}
if p.IsDay == nil || !*p.IsDay {
t.Fatalf("IsDay = %v, want true", p.IsDay)
}
if p.WindDirectionDegrees == nil || *p.WindDirectionDegrees != 225 {
t.Fatalf("WindDirectionDegrees = %v, want 225", p.WindDirectionDegrees)
}
if p.WindSpeedKmh == nil || math.Abs(*p.WindSpeedKmh-19.3128) > 0.001 {
t.Fatalf("WindSpeedKmh = %.6f, want ~19.3128", derefOrZero(p.WindSpeedKmh))
}
if p.ProbabilityOfPrecipitationPercent == nil || *p.ProbabilityOfPrecipitationPercent != 20 {
t.Fatalf("ProbabilityOfPrecipitationPercent = %v, want 20", p.ProbabilityOfPrecipitationPercent)
}
wantIssued := time.Date(2026, 3, 27, 15, 17, 1, 0, time.UTC)
if !run.IssuedAt.Equal(wantIssued) {
t.Fatalf("IssuedAt = %s, want %s", run.IssuedAt.Format(time.RFC3339), wantIssued.Format(time.RFC3339))
}
if !effectiveAt.Equal(wantIssued) {
t.Fatalf("effectiveAt = %s, want %s", effectiveAt.Format(time.RFC3339), wantIssued.Format(time.RFC3339))
}
assertNoLegacyForecastDescriptionKeys(t, p)
}
func TestBuildNarrativeForecastFallsBackToShortForecastDescription(t *testing.T) {
parsed := nwsNarrativeForecastResponse{}
parsed.Properties.GeneratedAt = "2026-03-27T15:17:01Z"
parsed.Properties.Periods = []nwsNarrativeForecastPeriod{
{
StartTime: "2026-03-27T18:00:00-05:00",
EndTime: "2026-03-28T06:00:00-05:00",
ShortForecast: " Mostly Clear ",
DetailedForecast: " ",
},
}
run, _, err := buildNarrativeForecast(parsed)
if err != nil {
t.Fatalf("buildNarrativeForecast() error = %v", err)
}
if len(run.Periods) != 1 {
t.Fatalf("periods len = %d, want 1", len(run.Periods))
}
if got, want := run.Periods[0].TextDescription, "Mostly Clear"; got != want {
t.Fatalf("TextDescription = %q, want %q", got, want)
}
}
func assertNoLegacyForecastDescriptionKeys(t *testing.T, period any) {
t.Helper()
b, err := json.Marshal(period)
if err != nil {
t.Fatalf("json.Marshal(period) error = %v", err)
}
var got map[string]any
if err := json.Unmarshal(b, &got); err != nil {
t.Fatalf("json.Unmarshal(period) error = %v", err)
}
for _, key := range []string{"conditionText", "providerRawDescription", "detailedText", "iconUrl"} {
if _, ok := got[key]; ok {
t.Fatalf("unexpected legacy key %q in marshaled period: %#v", key, got)
}
}
}
func derefOrZero(v *float64) float64 {
if v == nil {
return 0
}
return *v
}

View File

@@ -0,0 +1,235 @@
// FILE: internal/normalizers/nws/helpers.go
package nws
import (
"strconv"
"strings"
"unicode"
normcommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/common"
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
)
// centroidLatLon returns a best-effort centroid (lat, lon) from a GeoJSON polygon.
// If geometry is missing or malformed, returns (nil, nil).
func centroidLatLon(coords [][][]float64) (lat *float64, lon *float64) {
if len(coords) == 0 || len(coords[0]) == 0 {
return nil, nil
}
var sumLon, sumLat float64
var n float64
for _, pt := range coords[0] {
if len(pt) < 2 {
continue
}
sumLon += pt[0]
sumLat += pt[1]
n++
}
if n == 0 {
return nil, nil
}
avgLon := sumLon / n
avgLat := sumLat / n
return &avgLat, &avgLon
}
func tempCFromNWS(v *float64, unit string) *float64 {
if v == nil {
return nil
}
u := strings.ToUpper(strings.TrimSpace(unit))
switch u {
case "F":
c := normcommon.TempCFromF(*v)
return &c
case "C":
c := *v
return &c
default:
// Unknown unit; be conservative.
return nil
}
}
// wmoFromNWSForecast infers a canonical WMO code for a forecast period.
//
// Strategy:
// 1. Try to infer from shortForecast using the cross-provider fallback.
// 2. Special-case mixed rain+snow using temperature when available (since our WMO table
// does not include a “mixed precip” code).
// 3. Fall back to an icon token (e.g., "rain", "snow", "ovc", "bkn", "sct", ...).
func wmoFromNWSForecast(shortForecast, iconURL string, tempC *float64) model.WMOCode {
sf := strings.TrimSpace(shortForecast)
s := strings.ToLower(sf)
// Mixed precip heuristic: choose rain vs snow based on temperature.
if strings.Contains(s, "rain") && strings.Contains(s, "snow") {
if tempC != nil && *tempC <= 0.0 {
return 73 // Snow
}
return 63 // Rain
}
if code := normcommon.WMOFromTextDescription(sf); code != model.WMOUnknown {
return code
}
// Icon fallback: token is usually the last path segment (before any comma/query).
if token := nwsIconToken(iconURL); token != "" {
// Try the general text fallback first (works for "rain", "snow", etc.).
if code := normcommon.WMOFromTextDescription(token); code != model.WMOUnknown {
return code
}
// Sky-condition icon tokens are common; map conservatively.
switch token {
case "ovc", "bkn", "cloudy", "ovcast":
return 3
case "sct", "bkn-sct":
return 2
case "few":
return 1
case "skc", "clr", "clear":
return 0
}
}
return model.WMOUnknown
}
func nwsIconToken(iconURL string) string {
u := strings.TrimSpace(iconURL)
if u == "" {
return ""
}
// Drop query string.
base := strings.SplitN(u, "?", 2)[0]
// Take last path segment.
parts := strings.Split(base, "/")
if len(parts) == 0 {
return ""
}
last := parts[len(parts)-1]
if last == "" && len(parts) > 1 {
last = parts[len(parts)-2]
}
// Some icons look like "rain,30" or "snow,20".
last = strings.SplitN(last, ",", 2)[0]
last = strings.ToLower(strings.TrimSpace(last))
return last
}
// parseNWSWindSpeedKmh parses NWS wind speed strings like:
// - "9 mph"
// - "10 to 15 mph"
//
// and converts to km/h.
//
// Policy: if a range is present, we use the midpoint (best effort).
func parseNWSWindSpeedKmh(s string) *float64 {
raw := strings.ToLower(strings.TrimSpace(s))
if raw == "" {
return nil
}
nums := extractFloats(raw)
if len(nums) == 0 {
return nil
}
val := nums[0]
if len(nums) >= 2 && (strings.Contains(raw, " to ") || strings.Contains(raw, "-")) {
val = (nums[0] + nums[1]) / 2.0
}
switch {
case strings.Contains(raw, "mph"):
k := normcommon.SpeedKmhFromMph(val)
return &k
case strings.Contains(raw, "km/h") || strings.Contains(raw, "kph"):
k := val
return &k
case strings.Contains(raw, "kt") || strings.Contains(raw, "kts") || strings.Contains(raw, "knot"):
// 1 knot = 1.852 km/h
k := val * 1.852
return &k
default:
// Unknown unit; be conservative.
return nil
}
}
// parseNWSWindDirectionDegrees maps compass directions to degrees.
// Returns nil if direction is empty/unknown.
func parseNWSWindDirectionDegrees(dir string) *float64 {
d := strings.ToUpper(strings.TrimSpace(dir))
if d == "" {
return nil
}
// 16-wind compass.
m := map[string]float64{
"N": 0,
"NNE": 22.5,
"NE": 45,
"ENE": 67.5,
"E": 90,
"ESE": 112.5,
"SE": 135,
"SSE": 157.5,
"S": 180,
"SSW": 202.5,
"SW": 225,
"WSW": 247.5,
"W": 270,
"WNW": 292.5,
"NW": 315,
"NNW": 337.5,
}
if deg, ok := m[d]; ok {
return &deg
}
return nil
}
func extractFloats(s string) []float64 {
var out []float64
var buf strings.Builder
flush := func() {
if buf.Len() == 0 {
return
}
v, err := strconv.ParseFloat(buf.String(), 64)
if err == nil {
out = append(out, v)
}
buf.Reset()
}
for _, r := range s {
if unicode.IsDigit(r) || r == '.' {
buf.WriteRune(r)
continue
}
flush()
}
flush()
return out
}

View File

@@ -8,9 +8,10 @@ import (
"time" "time"
"gitea.maximumdirect.net/ejr/feedkit/event" "gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/model"
normcommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/common" normcommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/common"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/standards" nwscommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/nws"
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
) )
// ObservationNormalizer converts: // ObservationNormalizer converts:
@@ -46,19 +47,11 @@ func buildObservation(parsed nwsObservationResponse) (model.WeatherObservation,
// Timestamp (RFC3339) // Timestamp (RFC3339)
var ts time.Time var ts time.Time
if s := strings.TrimSpace(parsed.Properties.Timestamp); s != "" { if s := strings.TrimSpace(parsed.Properties.Timestamp); s != "" {
t, err := time.Parse(time.RFC3339, s) t, err := nwscommon.ParseTime(s)
if err != nil { if err != nil {
return model.WeatherObservation{}, time.Time{}, fmt.Errorf("nws observation normalize: invalid timestamp %q: %w", s, err) return model.WeatherObservation{}, time.Time{}, fmt.Errorf("invalid timestamp %q: %w", s, err)
} }
ts = t ts = t.UTC()
}
cloudLayers := make([]model.CloudLayer, 0, len(parsed.Properties.CloudLayers))
for _, cl := range parsed.Properties.CloudLayers {
cloudLayers = append(cloudLayers, model.CloudLayer{
BaseMeters: cl.Base.Value,
Amount: cl.Amount,
})
} }
// Preserve raw presentWeather objects (for troubleshooting / drift analysis). // Preserve raw presentWeather objects (for troubleshooting / drift analysis).
@@ -69,15 +62,25 @@ func buildObservation(parsed nwsObservationResponse) (model.WeatherObservation,
// Decode presentWeather into typed METAR phenomena for mapping. // Decode presentWeather into typed METAR phenomena for mapping.
phenomena := decodeMetarPhenomena(parsed.Properties.PresentWeather) phenomena := decodeMetarPhenomena(parsed.Properties.PresentWeather)
cloudLayers := parsed.Properties.CloudLayers
providerDesc := strings.TrimSpace(parsed.Properties.TextDescription) providerDesc := strings.TrimSpace(parsed.Properties.TextDescription)
// Determine canonical WMO condition code. // Determine canonical WMO condition code.
wmo := mapNWSToWMO(providerDesc, cloudLayers, phenomena) wmo := mapNWSToWMO(providerDesc, cloudLayers, phenomena)
// Canonical condition text comes from our WMO table. var isDay *bool
// NWS observation responses typically do not include a day/night flag -> nil. if lat, lon := observationLatLon(parsed.Geometry.Coordinates); lat != nil && lon != nil {
canonicalText := standards.WMOText(wmo, nil) isDay = isDayFromLatLonTime(*lat, *lon, ts)
}
// Apparent temperature: prefer wind chill when both are supplied.
var apparentC *float64
if parsed.Properties.WindChill.Value != nil {
apparentC = parsed.Properties.WindChill.Value
} else if parsed.Properties.HeatIndex.Value != nil {
apparentC = parsed.Properties.HeatIndex.Value
}
obs := model.WeatherObservation{ obs := model.WeatherObservation{
StationID: parsed.Properties.StationID, StationID: parsed.Properties.StationID,
@@ -85,15 +88,9 @@ func buildObservation(parsed nwsObservationResponse) (model.WeatherObservation,
Timestamp: ts, Timestamp: ts,
ConditionCode: wmo, ConditionCode: wmo,
ConditionText: canonicalText, IsDay: isDay,
IsDay: nil,
ProviderRawDescription: providerDesc, TextDescription: providerDesc,
// Transitional / human-facing:
// keep output consistent by populating TextDescription from canonical text.
TextDescription: canonicalText,
IconURL: parsed.Properties.Icon,
TemperatureC: parsed.Properties.Temperature.Value, TemperatureC: parsed.Properties.Temperature.Value,
DewpointC: parsed.Properties.Dewpoint.Value, DewpointC: parsed.Properties.Dewpoint.Value,
@@ -102,20 +99,21 @@ func buildObservation(parsed nwsObservationResponse) (model.WeatherObservation,
WindSpeedKmh: parsed.Properties.WindSpeed.Value, WindSpeedKmh: parsed.Properties.WindSpeed.Value,
WindGustKmh: parsed.Properties.WindGust.Value, WindGustKmh: parsed.Properties.WindGust.Value,
BarometricPressurePa: parsed.Properties.BarometricPressure.Value, BarometricPressurePa: pressurePrecedenceNWS(parsed.Properties.SeaLevelPressure.Value, parsed.Properties.BarometricPressure.Value),
SeaLevelPressurePa: parsed.Properties.SeaLevelPressure.Value,
VisibilityMeters: parsed.Properties.Visibility.Value, VisibilityMeters: parsed.Properties.Visibility.Value,
RelativeHumidityPercent: parsed.Properties.RelativeHumidity.Value, RelativeHumidityPercent: parsed.Properties.RelativeHumidity.Value,
WindChillC: parsed.Properties.WindChill.Value, ApparentTemperatureC: apparentC,
HeatIndexC: parsed.Properties.HeatIndex.Value,
ElevationMeters: parsed.Properties.Elevation.Value,
RawMessage: parsed.Properties.RawMessage,
PresentWeather: present, PresentWeather: present,
CloudLayers: cloudLayers,
} }
return obs, ts, nil return obs, ts, nil
} }
func pressurePrecedenceNWS(seaLevelPa, barometricPa *float64) *float64 {
if seaLevelPa != nil {
return seaLevelPa
}
return barometricPa
}

View File

@@ -0,0 +1,51 @@
package nws
import "testing"
func TestBuildObservationTextDescriptionAndPressurePrecedence(t *testing.T) {
barometric := 100200.0
seaLevel := 101400.0
parsed := nwsObservationResponse{}
parsed.Properties.Timestamp = "2026-03-16T19:00:00Z"
parsed.Properties.TextDescription = " Overcast "
parsed.Properties.BarometricPressure.Value = &barometric
parsed.Properties.SeaLevelPressure.Value = &seaLevel
obs, _, err := buildObservation(parsed)
if err != nil {
t.Fatalf("buildObservation() error = %v", err)
}
if got, want := obs.TextDescription, "Overcast"; got != want {
t.Fatalf("TextDescription = %q, want %q", got, want)
}
if obs.BarometricPressurePa == nil {
t.Fatalf("BarometricPressurePa = nil, want non-nil")
}
if got, want := *obs.BarometricPressurePa, seaLevel; got != want {
t.Fatalf("BarometricPressurePa = %v, want %v", got, want)
}
}
func TestBuildObservationPressureFallbackToBarometric(t *testing.T) {
barometric := 99900.0
parsed := nwsObservationResponse{}
parsed.Properties.Timestamp = "2026-03-16T19:00:00Z"
parsed.Properties.TextDescription = "Cloudy"
parsed.Properties.BarometricPressure.Value = &barometric
obs, _, err := buildObservation(parsed)
if err != nil {
t.Fatalf("buildObservation() error = %v", err)
}
if obs.BarometricPressurePa == nil {
t.Fatalf("BarometricPressurePa = nil, want non-nil")
}
if got, want := *obs.BarometricPressurePa, barometric; got != want {
t.Fatalf("BarometricPressurePa = %v, want %v", got, want)
}
}

View File

@@ -2,15 +2,17 @@
package nws package nws
import ( import (
fknormalize "gitea.maximumdirect.net/ejr/feedkit/normalize" fknormalize "gitea.maximumdirect.net/ejr/feedkit/processors/normalize"
) )
// Register registers NWS normalizers into the provided registry. var builtins = []fknormalize.Normalizer{
func Register(reg *fknormalize.Registry) { ObservationNormalizer{},
if reg == nil { ForecastNormalizer{},
return ForecastDiscussionNormalizer{},
} AlertsNormalizer{},
}
// Observations
reg.Register(ObservationNormalizer{}) // Register appends NWS normalizers in stable order.
func Register(in []fknormalize.Normalizer) []fknormalize.Normalizer {
return append(in, builtins...)
} }

View File

@@ -1,10 +1,18 @@
// FILE: ./internal/normalizers/nws/types.go // FILE: ./internal/normalizers/nws/types.go
package nws package nws
import (
"encoding/json"
)
// nwsObservationResponse is a minimal-but-sufficient representation of the NWS // nwsObservationResponse is a minimal-but-sufficient representation of the NWS
// station observation GeoJSON payload needed for mapping into model.WeatherObservation. // station observation GeoJSON payload needed for mapping into model.WeatherObservation.
type nwsObservationResponse struct { type nwsObservationResponse struct {
ID string `json:"id"` ID string `json:"id"`
Geometry struct {
Type string `json:"type"`
Coordinates []float64 `json:"coordinates"` // GeoJSON point: [lon, lat]
} `json:"geometry"`
Properties struct { Properties struct {
StationID string `json:"stationId"` StationID string `json:"stationId"`
StationName string `json:"stationName"` StationName string `json:"stationName"`
@@ -78,12 +86,185 @@ type nwsObservationResponse struct {
// We decode these as generic maps, then optionally interpret them in metar.go. // We decode these as generic maps, then optionally interpret them in metar.go.
PresentWeather []map[string]any `json:"presentWeather"` PresentWeather []map[string]any `json:"presentWeather"`
CloudLayers []struct { CloudLayers []nwsCloudLayer `json:"cloudLayers"`
Base struct {
UnitCode string `json:"unitCode"`
Value *float64 `json:"value"`
} `json:"base"`
Amount string `json:"amount"`
} `json:"cloudLayers"`
} `json:"properties"` } `json:"properties"`
} }
type nwsCloudLayer struct {
Base struct {
UnitCode string `json:"unitCode"`
Value *float64 `json:"value"`
} `json:"base"`
Amount string `json:"amount"`
}
// nwsHourlyForecastResponse is a minimal-but-sufficient representation of the NWS
// gridpoint hourly forecast GeoJSON payload needed for mapping into model.WeatherForecastRun.
type nwsHourlyForecastResponse struct {
Geometry struct {
Type string `json:"type"`
Coordinates [][][]float64 `json:"coordinates"` // GeoJSON polygon: [ring][point][lon,lat]
} `json:"geometry"`
Properties struct {
Units string `json:"units"` // "us" or "si" (often "us" for hourly)
ForecastGenerator string `json:"forecastGenerator"` // e.g. "HourlyForecastGenerator"
GeneratedAt string `json:"generatedAt"` // RFC3339-ish
UpdateTime string `json:"updateTime"` // RFC3339-ish
ValidTimes string `json:"validTimes"`
Elevation struct {
UnitCode string `json:"unitCode"`
Value *float64 `json:"value"`
} `json:"elevation"`
Periods []nwsHourlyForecastPeriod `json:"periods"`
} `json:"properties"`
}
type nwsHourlyForecastPeriod struct {
Number int `json:"number"`
Name string `json:"name"`
StartTime string `json:"startTime"`
EndTime string `json:"endTime"`
IsDaytime *bool `json:"isDaytime"`
Temperature *float64 `json:"temperature"`
TemperatureUnit string `json:"temperatureUnit"` // "F" or "C"
TemperatureTrend any `json:"temperatureTrend"`
ProbabilityOfPrecipitation struct {
UnitCode string `json:"unitCode"`
Value *float64 `json:"value"`
} `json:"probabilityOfPrecipitation"`
Dewpoint struct {
UnitCode string `json:"unitCode"`
Value *float64 `json:"value"`
} `json:"dewpoint"`
RelativeHumidity struct {
UnitCode string `json:"unitCode"`
Value *float64 `json:"value"`
} `json:"relativeHumidity"`
WindSpeed string `json:"windSpeed"` // e.g. "9 mph", "10 to 15 mph"
WindDirection string `json:"windDirection"` // e.g. "W", "NW"
Icon string `json:"icon"`
ShortForecast string `json:"shortForecast"`
DetailedForecast string `json:"detailedForecast"`
}
// nwsNarrativeForecastResponse is a minimal-but-sufficient representation of the NWS
// gridpoint narrative forecast GeoJSON payload needed for mapping into model.WeatherForecastRun.
type nwsNarrativeForecastResponse struct {
Geometry struct {
Type string `json:"type"`
Coordinates [][][]float64 `json:"coordinates"` // GeoJSON polygon: [ring][point][lon,lat]
} `json:"geometry"`
Properties struct {
Units string `json:"units"` // "us" or "si" (often "us" for narrative)
ForecastGenerator string `json:"forecastGenerator"` // e.g. "BaselineForecastGenerator"
GeneratedAt string `json:"generatedAt"` // RFC3339-ish
UpdateTime string `json:"updateTime"` // RFC3339-ish
ValidTimes string `json:"validTimes"`
Elevation struct {
UnitCode string `json:"unitCode"`
Value *float64 `json:"value"`
} `json:"elevation"`
Periods []nwsNarrativeForecastPeriod `json:"periods"`
} `json:"properties"`
}
type nwsNarrativeForecastPeriod struct {
Number int `json:"number"`
Name string `json:"name"`
StartTime string `json:"startTime"`
EndTime string `json:"endTime"`
IsDaytime *bool `json:"isDaytime"`
Temperature *float64 `json:"temperature"`
TemperatureUnit string `json:"temperatureUnit"` // "F" or "C"
TemperatureTrend any `json:"temperatureTrend"`
ProbabilityOfPrecipitation struct {
UnitCode string `json:"unitCode"`
Value *float64 `json:"value"`
} `json:"probabilityOfPrecipitation"`
WindSpeed string `json:"windSpeed"` // e.g. "9 mph", "10 to 15 mph"
WindDirection string `json:"windDirection"` // e.g. "W", "NW"
Icon string `json:"icon"`
ShortForecast string `json:"shortForecast"`
DetailedForecast string `json:"detailedForecast"`
}
// nwsAlertsResponse is a minimal-but-sufficient representation of the NWS /alerts
// FeatureCollection payload needed for mapping into model.WeatherAlertRun.
type nwsAlertsResponse struct {
Updated string `json:"updated"`
Title string `json:"title"`
Features []nwsAlertFeature `json:"features"`
}
type nwsAlertFeature struct {
ID string `json:"id"`
Properties nwsAlertProperties `json:"properties"`
}
type nwsAlertProperties struct {
// Identifiers.
ID string `json:"id"` // often a URL/URI
Identifier string `json:"identifier"` // CAP identifier string (sometimes distinct)
// Classification / headline.
Event string `json:"event"`
Headline string `json:"headline"`
Severity string `json:"severity"`
Urgency string `json:"urgency"`
Certainty string `json:"certainty"`
Status string `json:"status"`
MessageType string `json:"messageType"`
Category string `json:"category"`
Response string `json:"response"`
// Narrative.
Description string `json:"description"`
Instruction string `json:"instruction"`
// Timing.
Sent string `json:"sent"`
Effective string `json:"effective"`
Onset string `json:"onset"`
Expires string `json:"expires"`
Ends string `json:"ends"`
// Area / provenance.
AreaDesc string `json:"areaDesc"`
Sender string `json:"sender"`
SenderName string `json:"senderName"`
// Related alerts (updates/replacements).
// Keep flexible: some NWS payloads use an object array; others may degrade to strings.
References json.RawMessage `json:"references"`
}
type nwsAlertReference struct {
ID string `json:"id"`
Identifier string `json:"identifier"`
Sender string `json:"sender"`
Sent string `json:"sent"`
}

View File

@@ -4,8 +4,8 @@ package nws
import ( import (
"strings" "strings"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/model"
normcommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/common" normcommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/common"
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
) )
// mapNWSToWMO maps NWS signals into a canonical WMO code. // mapNWSToWMO maps NWS signals into a canonical WMO code.
@@ -14,7 +14,7 @@ import (
// 1. METAR phenomena (presentWeather) — most reliable for precip/hazards // 1. METAR phenomena (presentWeather) — most reliable for precip/hazards
// 2. textDescription keywords — weaker, but reusable across providers // 2. textDescription keywords — weaker, but reusable across providers
// 3. cloud layers fallback — only for sky-only conditions // 3. cloud layers fallback — only for sky-only conditions
func mapNWSToWMO(providerDesc string, cloudLayers []model.CloudLayer, phenomena []metarPhenomenon) model.WMOCode { func mapNWSToWMO(providerDesc string, cloudLayers []nwsCloudLayer, phenomena []metarPhenomenon) model.WMOCode {
// 1) Prefer METAR phenomena if present. // 1) Prefer METAR phenomena if present.
if code := wmoFromPhenomena(phenomena); code != model.WMOUnknown { if code := wmoFromPhenomena(phenomena); code != model.WMOUnknown {
return code return code
@@ -167,7 +167,7 @@ func wmoFromPhenomena(phenomena []metarPhenomenon) model.WMOCode {
return model.WMOUnknown return model.WMOUnknown
} }
func wmoFromCloudLayers(cloudLayers []model.CloudLayer) model.WMOCode { func wmoFromCloudLayers(cloudLayers []nwsCloudLayer) model.WMOCode {
// NWS cloud layer amount values commonly include: // NWS cloud layer amount values commonly include:
// OVC, BKN, SCT, FEW, SKC, CLR, VV (vertical visibility / obscured sky) // OVC, BKN, SCT, FEW, SKC, CLR, VV (vertical visibility / obscured sky)
// //

View File

@@ -1,19 +0,0 @@
// FILE: ./internal/normalizers/openmeteo/common.go
package openmeteo
import (
"time"
openmeteo "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/openmeteo"
)
// parseOpenMeteoTime parses Open-Meteo timestamps.
//
// The actual parsing logic lives in internal/providers/openmeteo so both the
// source (envelope EffectiveAt / event ID) and normalizer (canonical payload)
// can share identical timestamp behavior.
//
// We keep this thin wrapper to avoid churn in the normalizer package.
func parseOpenMeteoTime(s string, tz string, utcOffsetSeconds int) (time.Time, error) {
return openmeteo.ParseTime(s, tz, utcOffsetSeconds)
}

View File

@@ -0,0 +1,239 @@
package openmeteo
import (
"context"
"fmt"
"strings"
"time"
"gitea.maximumdirect.net/ejr/feedkit/event"
normcommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/common"
omcommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/openmeteo"
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
// ForecastNormalizer converts:
//
// standards.SchemaRawOpenMeteoHourlyForecastV1 -> standards.SchemaWeatherForecastV1
//
// It interprets Open-Meteo hourly forecast JSON and maps it into the canonical
// model.WeatherForecastRun representation.
//
// Caveats / assumptions:
// - Open-Meteo does not provide a true "issued at" timestamp; IssuedAt uses
// Event.EmittedAt when present, otherwise the first hourly time.
// - Hourly payloads are array-oriented; missing fields are treated as nil per-period.
// - Snowfall is provided in centimeters and is converted to millimeters.
// - apparent_temperature is mapped to ApparentTemperatureC when present.
type ForecastNormalizer struct{}
func (ForecastNormalizer) Match(e event.Event) bool {
return strings.TrimSpace(e.Schema) == standards.SchemaRawOpenMeteoHourlyForecastV1
}
func (ForecastNormalizer) Normalize(ctx context.Context, in event.Event) (*event.Event, error) {
_ = ctx // normalization is pure/CPU; keep ctx for future expensive steps
// If present, prefer the existing event EmittedAt as IssuedAt.
var fallbackIssued time.Time
if !in.EmittedAt.IsZero() {
fallbackIssued = in.EmittedAt.UTC()
}
return normcommon.NormalizeJSON(
in,
"openmeteo hourly forecast",
standards.SchemaWeatherForecastV1,
func(parsed omForecastResponse) (model.WeatherForecastRun, time.Time, error) {
return buildForecast(parsed, fallbackIssued)
},
)
}
// buildForecast contains the domain mapping logic (provider -> canonical model).
func buildForecast(parsed omForecastResponse, fallbackIssued time.Time) (model.WeatherForecastRun, time.Time, error) {
times := parsed.Hourly.Time
if len(times) == 0 {
return model.WeatherForecastRun{}, time.Time{}, fmt.Errorf("missing hourly.time")
}
issuedAt := fallbackIssued.UTC()
run := model.WeatherForecastRun{
LocationID: normcommon.SynthStationIDPtr("OPENMETEO", parsed.Latitude, parsed.Longitude),
LocationName: "Open-Meteo",
IssuedAt: time.Time{},
UpdatedAt: nil,
Product: model.ForecastProductHourly,
Latitude: floatCopy(parsed.Latitude),
Longitude: floatCopy(parsed.Longitude),
ElevationMeters: floatCopy(parsed.Elevation),
Periods: nil,
}
periods := make([]model.WeatherForecastPeriod, 0, len(times))
var prevStart time.Time
for i := range times {
start, err := parseHourlyTime(parsed, i)
if err != nil {
return model.WeatherForecastRun{}, time.Time{}, err
}
if issuedAt.IsZero() && i == 0 {
issuedAt = start
}
end, err := parsePeriodEnd(parsed, i, start, prevStart)
if err != nil {
return model.WeatherForecastRun{}, time.Time{}, err
}
prevStart = start
var isDay *bool
if v := intAt(parsed.Hourly.IsDay, i); v != nil {
b := *v == 1
isDay = &b
}
wmo := wmoAt(parsed.Hourly.WeatherCode, i)
canonicalText := standards.WMOText(wmo, isDay)
period := model.WeatherForecastPeriod{
StartTime: start,
EndTime: end,
Name: "",
IsDay: isDay,
ConditionCode: wmo,
TextDescription: canonicalText,
}
if v := floatAt(parsed.Hourly.Temperature2m, i); v != nil {
period.TemperatureC = v
}
if v := floatAt(parsed.Hourly.ApparentTemp, i); v != nil {
period.ApparentTemperatureC = v
}
if v := floatAt(parsed.Hourly.DewPoint2m, i); v != nil {
period.DewpointC = v
}
if v := floatAt(parsed.Hourly.RelativeHumidity2m, i); v != nil {
period.RelativeHumidityPercent = v
}
if v := floatAt(parsed.Hourly.WindDirection10m, i); v != nil {
period.WindDirectionDegrees = v
}
if v := floatAt(parsed.Hourly.WindSpeed10m, i); v != nil {
period.WindSpeedKmh = v
}
if v := floatAt(parsed.Hourly.WindGusts10m, i); v != nil {
period.WindGustKmh = v
}
if v := floatAt(parsed.Hourly.Visibility, i); v != nil {
period.VisibilityMeters = v
}
if v := floatAt(parsed.Hourly.CloudCover, i); v != nil {
period.CloudCoverPercent = v
}
if v := floatAt(parsed.Hourly.UVIndex, i); v != nil {
period.UVIndex = v
}
if v := floatAt(parsed.Hourly.PrecipProbability, i); v != nil {
period.ProbabilityOfPrecipitationPercent = v
}
if v := floatAt(parsed.Hourly.Precipitation, i); v != nil {
period.PrecipitationAmountMm = v
}
if v := floatAt(parsed.Hourly.Snowfall, i); v != nil {
mm := *v * 10.0
period.SnowfallDepthMM = &mm
}
if v := floatAt(parsed.Hourly.SurfacePressure, i); v != nil {
pa := normcommon.PressurePaFromHPa(*v)
period.BarometricPressurePa = &pa
} else if v := floatAt(parsed.Hourly.PressureMSL, i); v != nil {
pa := normcommon.PressurePaFromHPa(*v)
period.BarometricPressurePa = &pa
}
periods = append(periods, period)
}
run.Periods = periods
if issuedAt.IsZero() {
return model.WeatherForecastRun{}, time.Time{}, fmt.Errorf("missing issuedAt")
}
run.IssuedAt = issuedAt
// EffectiveAt policy for forecasts: treat IssuedAt as the effective time (dedupe-friendly).
return run, issuedAt, nil
}
func parseHourlyTime(parsed omForecastResponse, idx int) (time.Time, error) {
if idx < 0 || idx >= len(parsed.Hourly.Time) {
return time.Time{}, fmt.Errorf("hourly.time[%d] missing", idx)
}
raw := strings.TrimSpace(parsed.Hourly.Time[idx])
if raw == "" {
return time.Time{}, fmt.Errorf("hourly.time[%d] empty", idx)
}
t, err := omcommon.ParseTime(raw, parsed.Timezone, parsed.UTCOffsetSeconds)
if err != nil {
return time.Time{}, fmt.Errorf("hourly.time[%d] invalid %q: %w", idx, raw, err)
}
return t.UTC(), nil
}
func parsePeriodEnd(parsed omForecastResponse, idx int, start, prevStart time.Time) (time.Time, error) {
if idx+1 < len(parsed.Hourly.Time) {
return parseHourlyTime(parsed, idx+1)
}
step := time.Hour
if !prevStart.IsZero() {
if d := start.Sub(prevStart); d > 0 {
step = d
}
}
return start.Add(step), nil
}
func floatCopy(v *float64) *float64 {
if v == nil {
return nil
}
out := *v
return &out
}
func floatAt(vals []*float64, idx int) *float64 {
if idx < 0 || idx >= len(vals) {
return nil
}
return floatCopy(vals[idx])
}
func intAt(vals []*int, idx int) *int {
if idx < 0 || idx >= len(vals) {
return nil
}
if vals[idx] == nil {
return nil
}
out := *vals[idx]
return &out
}
func wmoAt(vals []*int, idx int) model.WMOCode {
if v := intAt(vals, idx); v != nil {
return model.WMOCode(*v)
}
return model.WMOUnknown
}

View File

@@ -0,0 +1,71 @@
package openmeteo
import (
"encoding/json"
"testing"
"time"
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
func TestBuildForecastUsesCanonicalTextDescription(t *testing.T) {
weatherCode := 2
isDay := 1
parsed := omForecastResponse{
Timezone: "UTC",
UTCOffsetSeconds: 0,
Hourly: omForecastHourly{
Time: []string{"2026-03-16T19:00"},
WeatherCode: []*int{&weatherCode},
IsDay: []*int{&isDay},
},
}
run, effectiveAt, err := buildForecast(parsed, time.Time{})
if err != nil {
t.Fatalf("buildForecast() error = %v", err)
}
if len(run.Periods) != 1 {
t.Fatalf("periods len = %d, want 1", len(run.Periods))
}
expectedText := standards.WMOText(model.WMOCode(weatherCode), boolPtr(true))
if got := run.Periods[0].TextDescription; got != expectedText {
t.Fatalf("TextDescription = %q, want %q", got, expectedText)
}
wantIssued := time.Date(2026, 3, 16, 19, 0, 0, 0, time.UTC)
if !run.IssuedAt.Equal(wantIssued) {
t.Fatalf("IssuedAt = %s, want %s", run.IssuedAt.Format(time.RFC3339), wantIssued.Format(time.RFC3339))
}
if !effectiveAt.Equal(wantIssued) {
t.Fatalf("effectiveAt = %s, want %s", effectiveAt.Format(time.RFC3339), wantIssued.Format(time.RFC3339))
}
assertNoLegacyForecastDescriptionKeys(t, run.Periods[0])
}
func boolPtr(v bool) *bool {
return &v
}
func assertNoLegacyForecastDescriptionKeys(t *testing.T, period any) {
t.Helper()
b, err := json.Marshal(period)
if err != nil {
t.Fatalf("json.Marshal(period) error = %v", err)
}
var got map[string]any
if err := json.Unmarshal(b, &got); err != nil {
t.Fatalf("json.Unmarshal(period) error = %v", err)
}
for _, key := range []string{"conditionText", "providerRawDescription", "detailedText", "iconUrl"} {
if _, ok := got[key]; ok {
t.Fatalf("unexpected legacy key %q in marshaled period: %#v", key, got)
}
}
}

View File

@@ -8,9 +8,10 @@ import (
"time" "time"
"gitea.maximumdirect.net/ejr/feedkit/event" "gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/model"
normcommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/common" normcommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/common"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/standards" omcommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/openmeteo"
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
) )
// ObservationNormalizer converts: // ObservationNormalizer converts:
@@ -54,9 +55,9 @@ func buildObservation(parsed omResponse) (model.WeatherObservation, time.Time, e
// Parse current.time. // Parse current.time.
var ts time.Time var ts time.Time
if s := strings.TrimSpace(parsed.Current.Time); s != "" { if s := strings.TrimSpace(parsed.Current.Time); s != "" {
t, err := parseOpenMeteoTime(s, parsed.Timezone, parsed.UTCOffsetSeconds) t, err := omcommon.ParseTime(s, parsed.Timezone, parsed.UTCOffsetSeconds)
if err != nil { if err != nil {
return model.WeatherObservation{}, time.Time{}, fmt.Errorf("openmeteo observation normalize: parse time %q: %w", s, err) return model.WeatherObservation{}, time.Time{}, fmt.Errorf("parse time %q: %w", s, err)
} }
ts = t.UTC() ts = t.UTC()
} }
@@ -77,38 +78,17 @@ func buildObservation(parsed omResponse) (model.WeatherObservation, time.Time, e
canonicalText := standards.WMOText(wmo, isDay) canonicalText := standards.WMOText(wmo, isDay)
// Station identity: Open-Meteo is not a station feed; synthesize from coordinates. // Station identity: Open-Meteo is not a station feed; synthesize from coordinates.
stationID := "" // Require BOTH lat/lon to avoid misleading OPENMETEO(0.00000,...) IDs.
if parsed.Latitude != nil || parsed.Longitude != nil { stationID := normcommon.SynthStationIDPtr("OPENMETEO", parsed.Latitude, parsed.Longitude)
lat := 0.0
lon := 0.0
if parsed.Latitude != nil {
lat = *parsed.Latitude
}
if parsed.Longitude != nil {
lon = *parsed.Longitude
}
stationID = fmt.Sprintf("OPENMETEO(%.5f,%.5f)", lat, lon)
}
obs := model.WeatherObservation{ obs := model.WeatherObservation{
StationID: stationID, StationID: stationID,
StationName: "Open-Meteo", StationName: "Open-Meteo",
Timestamp: ts, Timestamp: ts,
ConditionCode: wmo, ConditionCode: wmo,
ConditionText: canonicalText, IsDay: isDay,
IsDay: isDay,
// Open-Meteo does not provide a separate human text description for "current"
// when using weather_code; we leave provider evidence empty.
ProviderRawDescription: "",
// Transitional / human-facing:
// keep output consistent by populating TextDescription from canonical text.
TextDescription: canonicalText, TextDescription: canonicalText,
// IconURL: Open-Meteo does not provide an icon URL in this endpoint.
IconURL: "",
} }
// Measurements (all optional; only set when present). // Measurements (all optional; only set when present).
@@ -116,6 +96,10 @@ func buildObservation(parsed omResponse) (model.WeatherObservation, time.Time, e
v := *parsed.Current.Temperature2m v := *parsed.Current.Temperature2m
obs.TemperatureC = &v obs.TemperatureC = &v
} }
if parsed.Current.ApparentTemperature != nil {
v := *parsed.Current.ApparentTemperature
obs.ApparentTemperatureC = &v
}
if parsed.Current.RelativeHumidity2m != nil { if parsed.Current.RelativeHumidity2m != nil {
v := *parsed.Current.RelativeHumidity2m v := *parsed.Current.RelativeHumidity2m
@@ -137,20 +121,13 @@ func buildObservation(parsed omResponse) (model.WeatherObservation, time.Time, e
obs.WindGustKmh = &v obs.WindGustKmh = &v
} }
if parsed.Current.SurfacePressure != nil { if parsed.Current.PressureMSL != nil {
v := normcommon.PressurePaFromHPa(*parsed.Current.PressureMSL)
obs.BarometricPressurePa = &v
} else if parsed.Current.SurfacePressure != nil {
v := normcommon.PressurePaFromHPa(*parsed.Current.SurfacePressure) v := normcommon.PressurePaFromHPa(*parsed.Current.SurfacePressure)
obs.BarometricPressurePa = &v obs.BarometricPressurePa = &v
} }
if parsed.Current.PressureMSL != nil {
v := normcommon.PressurePaFromHPa(*parsed.Current.PressureMSL)
obs.SeaLevelPressurePa = &v
}
if parsed.Elevation != nil {
v := *parsed.Elevation
obs.ElevationMeters = &v
}
return obs, ts, nil return obs, ts, nil
} }

View File

@@ -0,0 +1,61 @@
package openmeteo
import "testing"
func TestBuildObservationTextDescriptionAndPressurePrecedence(t *testing.T) {
weatherCode := 2
pressureMSL := 1016.0
surfacePressure := 1009.0
parsed := omResponse{
Timezone: "UTC",
UTCOffsetSeconds: 0,
Current: omCurrent{
Time: "2026-03-16T19:00",
WeatherCode: &weatherCode,
PressureMSL: &pressureMSL,
SurfacePressure: &surfacePressure,
},
}
obs, _, err := buildObservation(parsed)
if err != nil {
t.Fatalf("buildObservation() error = %v", err)
}
if got, want := obs.TextDescription, "Partly Cloudy"; got != want {
t.Fatalf("TextDescription = %q, want %q", got, want)
}
if obs.BarometricPressurePa == nil {
t.Fatalf("BarometricPressurePa = nil, want non-nil")
}
if got, want := *obs.BarometricPressurePa, 101600.0; got != want {
t.Fatalf("BarometricPressurePa = %v, want %v", got, want)
}
}
func TestBuildObservationPressureFallbackToSurface(t *testing.T) {
surfacePressure := 1008.0
parsed := omResponse{
Timezone: "UTC",
UTCOffsetSeconds: 0,
Current: omCurrent{
Time: "2026-03-16T19:00",
SurfacePressure: &surfacePressure,
},
}
obs, _, err := buildObservation(parsed)
if err != nil {
t.Fatalf("buildObservation() error = %v", err)
}
if obs.BarometricPressurePa == nil {
t.Fatalf("BarometricPressurePa = nil, want non-nil")
}
if got, want := *obs.BarometricPressurePa, 100800.0; got != want {
t.Fatalf("BarometricPressurePa = %v, want %v", got, want)
}
}

View File

@@ -2,15 +2,15 @@
package openmeteo package openmeteo
import ( import (
fknormalize "gitea.maximumdirect.net/ejr/feedkit/normalize" fknormalize "gitea.maximumdirect.net/ejr/feedkit/processors/normalize"
) )
// Register registers Open-Meteo normalizers into the provided registry. var builtins = []fknormalize.Normalizer{
func Register(reg *fknormalize.Registry) { ObservationNormalizer{},
if reg == nil { ForecastNormalizer{},
return }
}
// Register appends Open-Meteo normalizers in stable order.
// Observations func Register(in []fknormalize.Normalizer) []fknormalize.Normalizer {
reg.Register(ObservationNormalizer{}) return append(in, builtins...)
} }

View File

@@ -18,9 +18,10 @@ type omResponse struct {
type omCurrent struct { type omCurrent struct {
Time string `json:"time"` // e.g. "2026-01-10T12:30" (often no timezone suffix) Time string `json:"time"` // e.g. "2026-01-10T12:30" (often no timezone suffix)
Temperature2m *float64 `json:"temperature_2m"` Temperature2m *float64 `json:"temperature_2m"`
RelativeHumidity2m *float64 `json:"relative_humidity_2m"` ApparentTemperature *float64 `json:"apparent_temperature"`
WeatherCode *int `json:"weather_code"` RelativeHumidity2m *float64 `json:"relative_humidity_2m"`
WeatherCode *int `json:"weather_code"`
WindSpeed10m *float64 `json:"wind_speed_10m"` // km/h (per Open-Meteo docs for these fields) WindSpeed10m *float64 `json:"wind_speed_10m"` // km/h (per Open-Meteo docs for these fields)
WindDirection10m *float64 `json:"wind_direction_10m"` // degrees WindDirection10m *float64 `json:"wind_direction_10m"` // degrees
@@ -31,3 +32,38 @@ type omCurrent struct {
IsDay *int `json:"is_day"` // 0/1 IsDay *int `json:"is_day"` // 0/1
} }
// omForecastResponse is a minimal-but-sufficient representation of Open-Meteo
// hourly forecast payloads for mapping into model.WeatherForecastRun.
type omForecastResponse struct {
Latitude *float64 `json:"latitude"`
Longitude *float64 `json:"longitude"`
Timezone string `json:"timezone"`
UTCOffsetSeconds int `json:"utc_offset_seconds"`
Elevation *float64 `json:"elevation"`
Hourly omForecastHourly `json:"hourly"`
}
type omForecastHourly struct {
Time []string `json:"time"`
Temperature2m []*float64 `json:"temperature_2m"`
RelativeHumidity2m []*float64 `json:"relative_humidity_2m"`
DewPoint2m []*float64 `json:"dew_point_2m"`
ApparentTemp []*float64 `json:"apparent_temperature"`
PrecipProbability []*float64 `json:"precipitation_probability"`
Precipitation []*float64 `json:"precipitation"`
Snowfall []*float64 `json:"snowfall"`
WeatherCode []*int `json:"weather_code"`
SurfacePressure []*float64 `json:"surface_pressure"`
PressureMSL []*float64 `json:"pressure_msl"`
WindSpeed10m []*float64 `json:"wind_speed_10m"`
WindDirection10m []*float64 `json:"wind_direction_10m"`
WindGusts10m []*float64 `json:"wind_gusts_10m"`
IsDay []*int `json:"is_day"`
CloudCover []*float64 `json:"cloud_cover"`
Visibility []*float64 `json:"visibility"`
UVIndex []*float64 `json:"uv_index"`
}

View File

@@ -4,6 +4,8 @@ package openweather
import ( import (
"fmt" "fmt"
"strings" "strings"
normcommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/common"
) )
// This file holds provider-specific helpers that are shared across multiple // This file holds provider-specific helpers that are shared across multiple
@@ -51,15 +53,6 @@ func inferIsDay(icon string, dt, sunrise, sunset int64) *bool {
return nil return nil
} }
// openWeatherIconURL builds the standard OpenWeather icon URL for the given icon code.
func openWeatherIconURL(icon string) string {
icon = strings.TrimSpace(icon)
if icon == "" {
return ""
}
return fmt.Sprintf("https://openweathermap.org/img/wn/%s@2x.png", icon)
}
// openWeatherStationID returns a stable station identifier for the given response. // openWeatherStationID returns a stable station identifier for the given response.
// Prefer the OpenWeather city ID when present; otherwise, fall back to coordinates. // Prefer the OpenWeather city ID when present; otherwise, fall back to coordinates.
func openWeatherStationID(parsed owmResponse) string { func openWeatherStationID(parsed owmResponse) string {
@@ -67,5 +60,5 @@ func openWeatherStationID(parsed owmResponse) string {
return fmt.Sprintf("OPENWEATHER(%d)", parsed.ID) return fmt.Sprintf("OPENWEATHER(%d)", parsed.ID)
} }
// Fallback: synthesize from coordinates. // Fallback: synthesize from coordinates.
return fmt.Sprintf("OPENWEATHER(%.5f,%.5f)", parsed.Coord.Lat, parsed.Coord.Lon) return normcommon.SynthStationID("OPENWEATHER", parsed.Coord.Lat, parsed.Coord.Lon)
} }

View File

@@ -7,9 +7,9 @@ import (
"time" "time"
"gitea.maximumdirect.net/ejr/feedkit/event" "gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/model"
normcommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/common" normcommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers/common"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/standards" "gitea.maximumdirect.net/ejr/weatherfeeder/model"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
) )
// ObservationNormalizer converts: // ObservationNormalizer converts:
@@ -68,12 +68,17 @@ func buildObservation(parsed owmResponse) (model.WeatherObservation, time.Time,
// - wind speed is m/s -> km/h conversion // - wind speed is m/s -> km/h conversion
tempC := parsed.Main.Temp tempC := parsed.Main.Temp
rh := parsed.Main.Humidity rh := parsed.Main.Humidity
var apparentC *float64
if parsed.Main.FeelsLike != nil {
v := *parsed.Main.FeelsLike
apparentC = &v
}
surfacePa := normcommon.PressurePaFromHPa(parsed.Main.Pressure) surfacePa := normcommon.PressurePaFromHPa(parsed.Main.Pressure)
var seaLevelPa *float64 barometricPa := &surfacePa
if parsed.Main.SeaLevel != nil { if parsed.Main.SeaLevel != nil {
v := normcommon.PressurePaFromHPa(*parsed.Main.SeaLevel) v := normcommon.PressurePaFromHPa(*parsed.Main.SeaLevel)
seaLevelPa = &v barometricPa = &v
} }
wsKmh := normcommon.SpeedKmhFromMps(parsed.Wind.Speed) wsKmh := normcommon.SpeedKmhFromMps(parsed.Wind.Speed)
@@ -91,9 +96,6 @@ func buildObservation(parsed owmResponse) (model.WeatherObservation, time.Time,
// Condition mapping: OpenWeather condition IDs -> canonical WMO code vocabulary. // Condition mapping: OpenWeather condition IDs -> canonical WMO code vocabulary.
wmo := mapOpenWeatherToWMO(owmID) wmo := mapOpenWeatherToWMO(owmID)
canonicalText := standards.WMOText(wmo, isDay)
iconURL := openWeatherIconURL(icon)
stationID := openWeatherStationID(parsed) stationID := openWeatherStationID(parsed)
stationName := strings.TrimSpace(parsed.Name) stationName := strings.TrimSpace(parsed.Name)
@@ -106,24 +108,18 @@ func buildObservation(parsed owmResponse) (model.WeatherObservation, time.Time,
StationName: stationName, StationName: stationName,
Timestamp: ts, Timestamp: ts,
ConditionCode: wmo, ConditionCode: wmo,
ConditionText: canonicalText, IsDay: isDay,
IsDay: isDay, TextDescription: rawDesc,
ProviderRawDescription: rawDesc, TemperatureC: &tempC,
ApparentTemperatureC: apparentC,
// Human-facing legacy fields: populate with canonical text for consistency.
TextDescription: canonicalText,
IconURL: iconURL,
TemperatureC: &tempC,
WindDirectionDegrees: parsed.Wind.Deg, WindDirectionDegrees: parsed.Wind.Deg,
WindSpeedKmh: &wsKmh, WindSpeedKmh: &wsKmh,
WindGustKmh: wgKmh, WindGustKmh: wgKmh,
BarometricPressurePa: &surfacePa, BarometricPressurePa: barometricPa,
SeaLevelPressurePa: seaLevelPa,
VisibilityMeters: visM, VisibilityMeters: visM,
RelativeHumidityPercent: &rh, RelativeHumidityPercent: &rh,

View File

@@ -0,0 +1,58 @@
package openweather
import "testing"
func TestBuildObservationTextDescriptionAndPressurePrecedence(t *testing.T) {
seaLevel := 1018.0
parsed := owmResponse{}
parsed.Dt = 1710000000
parsed.Main.Temp = 20.0
parsed.Main.Humidity = 45.0
parsed.Main.Pressure = 1000.0
parsed.Main.SeaLevel = &seaLevel
parsed.Wind.Speed = 3.0
parsed.Weather = []owmWeather{
{ID: 801, Main: "Clouds", Description: "few clouds", Icon: "02d"},
}
obs, _, err := buildObservation(parsed)
if err != nil {
t.Fatalf("buildObservation() error = %v", err)
}
if obs.TextDescription != "few clouds" {
t.Fatalf("TextDescription = %q, want %q", obs.TextDescription, "few clouds")
}
if obs.BarometricPressurePa == nil {
t.Fatalf("BarometricPressurePa = nil, want non-nil")
}
if got, want := *obs.BarometricPressurePa, 101800.0; got != want {
t.Fatalf("BarometricPressurePa = %v, want %v", got, want)
}
}
func TestBuildObservationPressureFallbackToSurface(t *testing.T) {
parsed := owmResponse{}
parsed.Dt = 1710000000
parsed.Main.Temp = 20.0
parsed.Main.Humidity = 45.0
parsed.Main.Pressure = 1001.0
parsed.Wind.Speed = 3.0
parsed.Weather = []owmWeather{
{ID: 800, Description: "clear sky", Icon: "01d"},
}
obs, _, err := buildObservation(parsed)
if err != nil {
t.Fatalf("buildObservation() error = %v", err)
}
if obs.BarometricPressurePa == nil {
t.Fatalf("BarometricPressurePa = nil, want non-nil")
}
if got, want := *obs.BarometricPressurePa, 100100.0; got != want {
t.Fatalf("BarometricPressurePa = %v, want %v", got, want)
}
}

View File

@@ -2,15 +2,14 @@
package openweather package openweather
import ( import (
fknormalize "gitea.maximumdirect.net/ejr/feedkit/normalize" fknormalize "gitea.maximumdirect.net/ejr/feedkit/processors/normalize"
) )
// Register registers OpenWeather normalizers into the provided registry. var builtins = []fknormalize.Normalizer{
func Register(reg *fknormalize.Registry) { ObservationNormalizer{},
if reg == nil { }
return
} // Register appends OpenWeather normalizers in stable order.
func Register(in []fknormalize.Normalizer) []fknormalize.Normalizer {
// Observations return append(in, builtins...)
reg.Register(ObservationNormalizer{})
} }

View File

@@ -15,10 +15,11 @@ type owmResponse struct {
Weather []owmWeather `json:"weather"` Weather []owmWeather `json:"weather"`
Main struct { Main struct {
Temp float64 `json:"temp"` // °C when units=metric (enforced by source) Temp float64 `json:"temp"` // °C when units=metric (enforced by source)
Pressure float64 `json:"pressure"` // hPa FeelsLike *float64 `json:"feels_like"` // °C when units=metric (enforced by source)
Humidity float64 `json:"humidity"` // % Pressure float64 `json:"pressure"` // hPa
SeaLevel *float64 `json:"sea_level"` Humidity float64 `json:"humidity"` // %
SeaLevel *float64 `json:"sea_level"`
} `json:"main"` } `json:"main"`
Visibility *float64 `json:"visibility"` // meters (optional) Visibility *float64 `json:"visibility"` // meters (optional)

View File

@@ -1,7 +1,7 @@
// FILE: ./internal/normalizers/openweather/wmo_map.go // FILE: ./internal/normalizers/openweather/wmo_map.go
package openweather package openweather
import "gitea.maximumdirect.net/ejr/weatherfeeder/internal/model" import "gitea.maximumdirect.net/ejr/weatherfeeder/model"
// mapOpenWeatherToWMO maps OpenWeather weather condition IDs into weatherfeeder's // mapOpenWeatherToWMO maps OpenWeather weather condition IDs into weatherfeeder's
// canonical WMO code vocabulary. // canonical WMO code vocabulary.

View File

@@ -0,0 +1,8 @@
// Package nws contains provider-specific helper code for the National Weather Service
// used by both sources and normalizers.
//
// Rules:
// - No network I/O here (sources fetch; normalizers transform).
// - Keep helpers deterministic and easy to unit test.
// - Prefer putting provider quirks/parsing here when sources + normalizers both need it.
package nws

View File

@@ -0,0 +1,552 @@
package nws
import (
"fmt"
"html"
"regexp"
"strconv"
"strings"
"time"
)
type ForecastDiscussion struct {
OfficeID string
OfficeName string
Product string
IssuedAt time.Time
UpdatedAt *time.Time
KeyMessages []string
ShortTerm *ForecastDiscussionSection
LongTerm *ForecastDiscussionSection
}
type ForecastDiscussionSection struct {
Qualifier string
IssuedAt *time.Time
Text string
}
var (
forecastDiscussionHeaderRE = regexp.MustCompile(`^\.(KEY MESSAGES|SHORT TERM|LONG TERM|AVIATION)\.\.\.(.*)$`)
forecastDiscussionAFDRE = regexp.MustCompile(`^AFD([A-Z]{3})$`)
forecastDiscussionWMORE = regexp.MustCompile(`\bK([A-Z]{3})\b`)
forecastDiscussionSigRE = regexp.MustCompile(`^[A-Z]{2,6}$`)
)
func ParseForecastDiscussionHTML(raw string) (ForecastDiscussion, error) {
text, err := ExtractForecastDiscussionText(raw)
if err != nil {
return ForecastDiscussion{}, err
}
parsed, err := ParseForecastDiscussionText(text)
if err != nil {
return ForecastDiscussion{}, err
}
parsed.UpdatedAt = parseForecastDiscussionUpdatedAt(raw)
return parsed, nil
}
func ExtractForecastDiscussionText(raw string) (string, error) {
lower := strings.ToLower(raw)
searchFrom := 0
for {
openStart := strings.Index(lower[searchFrom:], "<pre")
if openStart < 0 {
return "", fmt.Errorf("missing <pre class=\"glossaryProduct\"> block")
}
openStart += searchFrom
openEnd := strings.Index(lower[openStart:], ">")
if openEnd < 0 {
return "", fmt.Errorf("unterminated <pre> tag")
}
openEnd += openStart
tag := lower[openStart : openEnd+1]
if isGlossaryProductTag(tag) {
closeStart := strings.Index(lower[openEnd+1:], "</pre>")
if closeStart < 0 {
return "", fmt.Errorf("missing closing </pre> for glossaryProduct block")
}
closeStart += openEnd + 1
text := html.UnescapeString(raw[openEnd+1 : closeStart])
text = strings.ReplaceAll(text, "\r\n", "\n")
text = strings.ReplaceAll(text, "\r", "\n")
return text, nil
}
searchFrom = openEnd + 1
}
}
func ParseForecastDiscussionText(text string) (ForecastDiscussion, error) {
lines := splitLines(text)
officeID := parseForecastDiscussionOfficeID(lines)
officeName, issuedAt, err := parseForecastDiscussionHeader(lines)
if err != nil {
return ForecastDiscussion{}, err
}
out := ForecastDiscussion{
OfficeID: officeID,
OfficeName: officeName,
Product: "afd",
IssuedAt: issuedAt.UTC(),
}
if block, ok := extractForecastDiscussionSection(lines, "KEY MESSAGES"); ok {
out.KeyMessages = parseForecastDiscussionKeyMessages(block)
}
if block, ok := extractForecastDiscussionSection(lines, "SHORT TERM"); ok {
section, err := parseForecastDiscussionTextSection(block)
if err != nil {
return ForecastDiscussion{}, fmt.Errorf("parse SHORT TERM: %w", err)
}
out.ShortTerm = &section
}
if block, ok := extractForecastDiscussionSection(lines, "LONG TERM"); ok {
section, err := parseForecastDiscussionTextSection(block)
if err != nil {
return ForecastDiscussion{}, fmt.Errorf("parse LONG TERM: %w", err)
}
out.LongTerm = &section
}
return out, nil
}
func isGlossaryProductTag(tag string) bool {
tag = strings.ToLower(tag)
return strings.Contains(tag, `class="glossaryproduct"`) ||
strings.Contains(tag, `class='glossaryproduct'`) ||
strings.Contains(tag, `class="glossaryproduct `) ||
strings.Contains(tag, `class='glossaryproduct `)
}
func parseForecastDiscussionUpdatedAt(raw string) *time.Time {
lower := strings.ToLower(raw)
searchFrom := 0
for {
metaStart := strings.Index(lower[searchFrom:], "<meta")
if metaStart < 0 {
return nil
}
metaStart += searchFrom
metaEnd := strings.Index(lower[metaStart:], ">")
if metaEnd < 0 {
return nil
}
metaEnd += metaStart
tag := raw[metaStart : metaEnd+1]
if !strings.EqualFold(strings.TrimSpace(extractHTMLAttr(tag, "name")), "DC.date.created") {
searchFrom = metaEnd + 1
continue
}
content := strings.TrimSpace(extractHTMLAttr(tag, "content"))
if content == "" {
return nil
}
t, err := ParseTime(content)
if err != nil {
return nil
}
tt := t.UTC()
return &tt
}
}
func extractHTMLAttr(tag, attr string) string {
lower := strings.ToLower(tag)
attrLower := strings.ToLower(attr)
for i := 0; i < len(lower); i++ {
idx := strings.Index(lower[i:], attrLower)
if idx < 0 {
return ""
}
idx += i
if idx > 0 {
prev := lower[idx-1]
if isAttrNameChar(prev) {
i = idx + len(attrLower)
continue
}
}
j := idx + len(attrLower)
for j < len(lower) && isHTMLSpace(lower[j]) {
j++
}
if j >= len(lower) || lower[j] != '=' {
i = idx + len(attrLower)
continue
}
j++
for j < len(lower) && isHTMLSpace(lower[j]) {
j++
}
if j >= len(tag) {
return ""
}
quote := tag[j]
if quote != '"' && quote != '\'' {
return ""
}
j++
k := j
for k < len(tag) && tag[k] != quote {
k++
}
if k >= len(tag) {
return ""
}
return html.UnescapeString(tag[j:k])
}
return ""
}
func isHTMLSpace(b byte) bool {
switch b {
case ' ', '\n', '\r', '\t', '\f':
return true
default:
return false
}
}
func isAttrNameChar(b byte) bool {
switch {
case b >= 'a' && b <= 'z':
return true
case b >= 'A' && b <= 'Z':
return true
case b >= '0' && b <= '9':
return true
case b == '-' || b == '_' || b == ':':
return true
default:
return false
}
}
func splitLines(text string) []string {
text = strings.ReplaceAll(text, "\r\n", "\n")
text = strings.ReplaceAll(text, "\r", "\n")
return strings.Split(text, "\n")
}
func parseForecastDiscussionOfficeID(lines []string) string {
for _, raw := range lines {
line := strings.TrimSpace(raw)
if m := forecastDiscussionAFDRE.FindStringSubmatch(line); len(m) == 2 {
return m[1]
}
}
for _, raw := range lines {
line := strings.TrimSpace(raw)
if m := forecastDiscussionWMORE.FindStringSubmatch(line); len(m) == 2 {
return m[1]
}
}
return ""
}
func parseForecastDiscussionHeader(lines []string) (string, time.Time, error) {
for i, raw := range lines {
line := strings.TrimSpace(raw)
if !strings.HasPrefix(line, "National Weather Service ") {
continue
}
officeName := line
for j := i + 1; j < len(lines); j++ {
tsLine := strings.TrimSpace(lines[j])
if tsLine == "" {
continue
}
issuedAt, err := parseForecastDiscussionIssueTime(tsLine)
if err != nil {
return "", time.Time{}, fmt.Errorf("parse bulletin issuedAt %q: %w", tsLine, err)
}
return officeName, issuedAt.UTC(), nil
}
return "", time.Time{}, fmt.Errorf("missing bulletin issue time after office line")
}
return "", time.Time{}, fmt.Errorf("missing office header")
}
func parseForecastDiscussionIssueTime(line string) (time.Time, error) {
line = strings.TrimSpace(line)
line = strings.TrimPrefix(line, "Issued at ")
line = strings.TrimSpace(line)
parts := strings.Fields(line)
if len(parts) != 7 {
return time.Time{}, fmt.Errorf("unexpected issue time format")
}
loc, err := forecastDiscussionLocation(parts[2])
if err != nil {
return time.Time{}, err
}
datePart, err := time.Parse("Mon Jan 2 2006", strings.Join(parts[3:], " "))
if err != nil {
return time.Time{}, err
}
hour, minute, err := parseForecastDiscussionClock(parts[0], parts[1])
if err != nil {
return time.Time{}, err
}
return time.Date(
datePart.Year(),
datePart.Month(),
datePart.Day(),
hour,
minute,
0,
0,
loc,
), nil
}
func parseForecastDiscussionClock(rawClock, rawAMPM string) (int, int, error) {
clock := strings.TrimSpace(rawClock)
ampm := strings.ToUpper(strings.TrimSpace(rawAMPM))
if ampm != "AM" && ampm != "PM" {
return 0, 0, fmt.Errorf("unexpected meridiem %q", rawAMPM)
}
n, err := strconv.Atoi(clock)
if err != nil {
return 0, 0, fmt.Errorf("invalid clock %q", rawClock)
}
hour := n
minute := 0
if len(clock) >= 3 {
hour = n / 100
minute = n % 100
}
if hour < 1 || hour > 12 {
return 0, 0, fmt.Errorf("invalid hour %q", rawClock)
}
if minute < 0 || minute > 59 {
return 0, 0, fmt.Errorf("invalid minute %q", rawClock)
}
if ampm == "AM" {
if hour == 12 {
hour = 0
}
return hour, minute, nil
}
if hour != 12 {
hour += 12
}
return hour, minute, nil
}
func forecastDiscussionLocation(abbrev string) (*time.Location, error) {
offsets := map[string]int{
"AST": -4 * 3600,
"ADT": -3 * 3600,
"EST": -5 * 3600,
"EDT": -4 * 3600,
"CST": -6 * 3600,
"CDT": -5 * 3600,
"MST": -7 * 3600,
"MDT": -6 * 3600,
"PST": -8 * 3600,
"PDT": -7 * 3600,
"AKST": -9 * 3600,
"AKDT": -8 * 3600,
"HST": -10 * 3600,
"UTC": 0,
"GMT": 0,
}
abbr := strings.ToUpper(strings.TrimSpace(abbrev))
offset, ok := offsets[abbr]
if !ok {
return nil, fmt.Errorf("unsupported time zone %q", abbrev)
}
return time.FixedZone(abbr, offset), nil
}
func extractForecastDiscussionSection(lines []string, section string) ([]string, bool) {
target := "." + section + "..."
for i, raw := range lines {
line := strings.TrimSpace(raw)
if !strings.HasPrefix(line, target) {
continue
}
out := []string{line}
for j := i + 1; j < len(lines); j++ {
next := strings.TrimSpace(lines[j])
if next == "&&" || next == "$$" || strings.Contains(next, "WATCHES/WARNINGS/ADVISORIES") {
break
}
if j > i+1 && isForecastDiscussionSectionHeader(next) {
break
}
out = append(out, lines[j])
}
return out, true
}
return nil, false
}
func isForecastDiscussionSectionHeader(line string) bool {
return forecastDiscussionHeaderRE.MatchString(strings.TrimSpace(line))
}
func parseForecastDiscussionKeyMessages(block []string) []string {
if len(block) <= 1 {
return nil
}
body := trimBlankLines(block[1:])
var messages []string
var current strings.Builder
flush := func() {
msg := strings.TrimSpace(current.String())
if msg != "" {
messages = append(messages, msg)
}
current.Reset()
}
for _, raw := range body {
line := strings.TrimSpace(raw)
if line == "" {
continue
}
if strings.HasPrefix(line, "-") {
flush()
line = strings.TrimSpace(strings.TrimPrefix(line, "-"))
current.WriteString(line)
continue
}
if current.Len() > 0 {
current.WriteByte(' ')
}
current.WriteString(line)
}
flush()
return messages
}
func parseForecastDiscussionTextSection(block []string) (ForecastDiscussionSection, error) {
if len(block) == 0 {
return ForecastDiscussionSection{}, fmt.Errorf("empty section")
}
section := ForecastDiscussionSection{
Qualifier: parseForecastDiscussionQualifier(strings.TrimSpace(block[0])),
}
body := trimBlankLines(block[1:])
if len(body) == 0 {
return section, nil
}
first := strings.TrimSpace(body[0])
if strings.HasPrefix(first, "Issued at ") {
issuedAt, err := parseForecastDiscussionIssueTime(first)
if err != nil {
return ForecastDiscussionSection{}, fmt.Errorf("parse section issuedAt %q: %w", first, err)
}
tt := issuedAt.UTC()
section.IssuedAt = &tt
body = trimBlankLines(body[1:])
}
body = trimForecastDiscussionSignatureLines(body)
section.Text = joinForecastDiscussionParagraphs(body)
return section, nil
}
func parseForecastDiscussionQualifier(header string) string {
m := forecastDiscussionHeaderRE.FindStringSubmatch(header)
if len(m) != 3 {
return ""
}
return strings.TrimSpace(m[2])
}
func trimBlankLines(lines []string) []string {
start := 0
for start < len(lines) && strings.TrimSpace(lines[start]) == "" {
start++
}
end := len(lines)
for end > start && strings.TrimSpace(lines[end-1]) == "" {
end--
}
return lines[start:end]
}
func trimForecastDiscussionSignatureLines(lines []string) []string {
lines = trimBlankLines(lines)
for len(lines) > 0 {
last := strings.TrimSpace(lines[len(lines)-1])
if last == "" {
lines = lines[:len(lines)-1]
continue
}
if forecastDiscussionSigRE.MatchString(last) {
lines = trimBlankLines(lines[:len(lines)-1])
continue
}
break
}
return lines
}
func joinForecastDiscussionParagraphs(lines []string) string {
lines = trimBlankLines(lines)
if len(lines) == 0 {
return ""
}
var paragraphs []string
current := make([]string, 0, len(lines))
flush := func() {
if len(current) == 0 {
return
}
paragraphs = append(paragraphs, strings.Join(current, " "))
current = current[:0]
}
for _, raw := range lines {
line := strings.TrimSpace(raw)
if line == "" {
flush()
continue
}
current = append(current, line)
}
flush()
return strings.Join(paragraphs, "\n\n")
}

View File

@@ -0,0 +1,118 @@
package nws
import (
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func TestParseForecastDiscussionHTMLParsesExpectedFields(t *testing.T) {
raw := loadForecastDiscussionSampleHTML(t)
got, err := ParseForecastDiscussionHTML(raw)
if err != nil {
t.Fatalf("ParseForecastDiscussionHTML() error = %v", err)
}
if got.OfficeID != "LSX" {
t.Fatalf("OfficeID = %q, want LSX", got.OfficeID)
}
if got.OfficeName != "National Weather Service Saint Louis MO" {
t.Fatalf("OfficeName = %q", got.OfficeName)
}
if got.Product != "afd" {
t.Fatalf("Product = %q, want afd", got.Product)
}
wantIssuedAt := time.Date(2026, 3, 28, 19, 24, 0, 0, time.UTC)
if !got.IssuedAt.Equal(wantIssuedAt) {
t.Fatalf("IssuedAt = %s, want %s", got.IssuedAt.Format(time.RFC3339), wantIssuedAt.Format(time.RFC3339))
}
wantUpdatedAt := time.Date(2026, 3, 28, 20, 29, 47, 0, time.UTC)
if got.UpdatedAt == nil || !got.UpdatedAt.Equal(wantUpdatedAt) {
t.Fatalf("UpdatedAt = %v, want %s", got.UpdatedAt, wantUpdatedAt.Format(time.RFC3339))
}
wantMessages := []string{
"Elevated fire danger conditions are expected across a broad area tomorrow afternoon due to breezy southwest winds and low humidity.",
"Very warm temperatures are expected once again Monday and Tuesday, with highs well into the 80s.",
"A cold front late Tuesday or early Wednesday brings our next chance of thunderstorms, followed by a cooldown and possibly more chances for rain later in the week.",
}
if len(got.KeyMessages) != len(wantMessages) {
t.Fatalf("KeyMessages len = %d, want %d", len(got.KeyMessages), len(wantMessages))
}
for i := range wantMessages {
if got.KeyMessages[i] != wantMessages[i] {
t.Fatalf("KeyMessages[%d] = %q, want %q", i, got.KeyMessages[i], wantMessages[i])
}
}
if got.ShortTerm == nil {
t.Fatalf("ShortTerm is nil")
}
if got.ShortTerm.Qualifier != "(Through Late Sunday Night)" {
t.Fatalf("ShortTerm.Qualifier = %q", got.ShortTerm.Qualifier)
}
if got.ShortTerm.IssuedAt == nil || !got.ShortTerm.IssuedAt.Equal(time.Date(2026, 3, 28, 19, 19, 0, 0, time.UTC)) {
t.Fatalf("ShortTerm.IssuedAt = %v", got.ShortTerm.IssuedAt)
}
if !strings.Contains(got.ShortTerm.Text, "After a chilly morning") {
t.Fatalf("ShortTerm.Text missing expected prose: %q", got.ShortTerm.Text)
}
if strings.Contains(got.ShortTerm.Text, "BRC") {
t.Fatalf("ShortTerm.Text should not include signature: %q", got.ShortTerm.Text)
}
if strings.Contains(got.ShortTerm.Text, "\n\n\n") {
t.Fatalf("ShortTerm.Text contains unexpected paragraph breaks: %q", got.ShortTerm.Text)
}
if got.LongTerm == nil {
t.Fatalf("LongTerm is nil")
}
if got.LongTerm.Qualifier != "(Monday through Next Saturday)" {
t.Fatalf("LongTerm.Qualifier = %q", got.LongTerm.Qualifier)
}
if got.LongTerm.IssuedAt == nil || !got.LongTerm.IssuedAt.Equal(time.Date(2026, 3, 28, 19, 19, 0, 0, time.UTC)) {
t.Fatalf("LongTerm.IssuedAt = %v", got.LongTerm.IssuedAt)
}
if !strings.Contains(got.LongTerm.Text, "The peak of the warmth arrives Monday and Tuesday") {
t.Fatalf("LongTerm.Text missing expected prose: %q", got.LongTerm.Text)
}
if strings.Contains(got.LongTerm.Text, "AVIATION") || strings.Contains(got.LongTerm.Text, "WATCHES/WARNINGS/ADVISORIES") {
t.Fatalf("LongTerm.Text includes content from other sections: %q", got.LongTerm.Text)
}
}
func TestParseForecastDiscussionHTMLMissingPreBlock(t *testing.T) {
_, err := ParseForecastDiscussionHTML("<html><body><div>no pre block</div></body></html>")
if err == nil {
t.Fatalf("ParseForecastDiscussionHTML() error = nil, want error")
}
if !strings.Contains(err.Error(), "glossaryProduct") {
t.Fatalf("error = %q, want glossaryProduct context", err)
}
}
func TestParseForecastDiscussionTextMissingIssueTime(t *testing.T) {
_, err := ParseForecastDiscussionText("National Weather Service Saint Louis MO\n\n.KEY MESSAGES...\n- Test")
if err == nil {
t.Fatalf("ParseForecastDiscussionText() error = nil, want error")
}
if !strings.Contains(err.Error(), "issue time") {
t.Fatalf("error = %q, want issue time context", err)
}
}
func loadForecastDiscussionSampleHTML(t *testing.T) string {
t.Helper()
path := filepath.Join("testdata", "forecast_discussion_sample.html")
b, err := os.ReadFile(path)
if err != nil {
t.Fatalf("os.ReadFile(%q) error = %v", path, err)
}
return string(b)
}

View File

@@ -0,0 +1,84 @@
<!DOCTYPE html>
<html class="no-js">
<head>
<meta name="DC.date.created" scheme="ISO8601" content="2026-03-28T20:29:47+00:00" />
<title>National Weather Service</title>
</head>
<body>
<pre class="glossaryProduct">
988
FXUS63 KLSX 281924
AFDLSX
Area Forecast Discussion
National Weather Service Saint Louis MO
224 PM CDT Sat Mar 28 2026
.KEY MESSAGES...
- Elevated fire danger conditions are expected across a broad area
tomorrow afternoon due to breezy southwest winds and low
humidity.
- Very warm temperatures are expected once again Monday and
Tuesday, with highs well into the 80s.
- A cold front late Tuesday or early Wednesday brings our next
chance of thunderstorms, followed by a cooldown and possibly
more chances for rain later in the week.
&&
.SHORT TERM... (Through Late Sunday Night)
Issued at 219 PM CDT Sat Mar 28 2026
After a chilly morning that saw widespread freezing temperatures,
another warmup is in store over the next several days as southerly
winds become re-established. We will also see the return of
shower/thunderstorm chances Tuesday onward as we enter a more
unsettled pattern.
In the near-term, the focus continues to be on some lingering fire
weather potential thanks to the presence of an exceptionally dry
airmass.
BRC
&&
.LONG TERM... (Monday through Next Saturday)
Issued at 219 PM CDT Sat Mar 28 2026
The peak of the warmth arrives Monday and Tuesday, as a broad, but
low-amplitude ridge nudges eastward and steady warm/moist advection
continues on both days.
Wednesday onward, the day-to-day details become much less clear, but
latest trends suggest that an active/wet pattern will likely
continue as another more substantial trough follows with additional
chances for showers/thunderstorms late in the week.
BRC
&&
.AVIATION... (For the 18z TAFs through 18z Sunday Afternoon)
Issued at 1133 AM CDT Sat Mar 28 2026
VFR conditions are expected throughout the 18Z TAF period.
BRC
&&
.LSX WATCHES/WARNINGS/ADVISORIES...
MO...None.
IL...None.
&&
$$
WFO LSX
</pre>
</body>
</html>

View File

@@ -0,0 +1,50 @@
package nws
import (
"fmt"
"strings"
"time"
)
// ParseTime parses NWS timestamps.
//
// NWS observation timestamps are typically RFC3339, sometimes with fractional seconds.
// We accept RFC3339Nano first, then RFC3339.
func ParseTime(s string) (time.Time, error) {
s = strings.TrimSpace(s)
if s == "" {
return time.Time{}, fmt.Errorf("empty time")
}
if t, err := time.Parse(time.RFC3339Nano, s); err == nil {
return t, nil
}
if t, err := time.Parse(time.RFC3339, s); err == nil {
return t, nil
}
return time.Time{}, fmt.Errorf("unsupported NWS timestamp format: %q", s)
}
// ParseTimeBestEffort parses an NWS timestamp and returns it in UTC.
//
// This is a convenience for normalizers that want "best effort" parsing:
// invalid/empty strings do not fail the entire normalization; they return zero time.
func ParseTimeBestEffort(s string) time.Time {
t, err := ParseTime(s)
if err != nil {
return time.Time{}
}
return t.UTC()
}
// ParseTimePtr parses an NWS timestamp and returns a UTC *time.Time.
//
// Empty/unparseable input returns nil. This is useful for optional CAP fields.
func ParseTimePtr(s string) *time.Time {
t := ParseTimeBestEffort(s)
if t.IsZero() {
return nil
}
return &t
}

View File

@@ -0,0 +1,8 @@
// Package openweather contains provider-specific helper code for OpenWeather used by
// both sources and normalizers.
//
// Rules:
// - No network I/O here.
// - Keep helpers deterministic and easy to unit test.
// - Put provider invariants here (e.g., units=metric enforcement).
package openweather

View File

@@ -0,0 +1,26 @@
package openweather
import (
"fmt"
"net/url"
"strings"
)
// RequireMetricUnits enforces weatherfeeder's OpenWeather invariant:
// the request URL must include units=metric (otherwise temperatures/winds/pressure differ).
func RequireMetricUnits(rawURL string) error {
u, err := url.Parse(strings.TrimSpace(rawURL))
if err != nil {
return fmt.Errorf("invalid url %q: %w", rawURL, err)
}
units := strings.ToLower(strings.TrimSpace(u.Query().Get("units")))
if units != "metric" {
if units == "" {
units = "(missing; defaults to standard)"
}
return fmt.Errorf("url must include units=metric (got units=%s)", units)
}
return nil
}

View File

@@ -0,0 +1,188 @@
// Package postgres documents weatherfeeder's PostgreSQL sink contract for
// downstream SQL consumers.
//
// This package wires weatherfeeder canonical events into normalized relational
// tables. Downstream consumers can reconstruct the same canonical JSON objects
// that were written by joining parent/child tables as described below.
//
// Canonical input schemas:
// - weather.observation.v1 -> model.WeatherObservation
// - weather.forecast.v1 -> model.WeatherForecastRun
// - weather.alert.v1 -> model.WeatherAlertRun
//
// Parent/child relationships:
// - observations.event_id -> observation_present_weather.event_id
// - forecasts.event_id -> forecast_periods.run_event_id
// - alert_runs.event_id -> alerts.run_event_id
// - alerts.(run_event_id, alert_index) -> alert_references.(run_event_id, alert_index)
//
// Dedupe and retention behavior:
// - Parent primary keys (event_id): observations, forecasts, alert_runs.
// - Child primary keys use positional indexes to preserve payload order.
// - Prune columns:
// - observations.observed_at
// - observation_present_weather.observed_at
// - forecasts.issued_at
// - forecast_periods.issued_at
// - alert_runs.as_of
// - alerts.as_of
// - alert_references.as_of
//
// Envelope field mapping (shared parent columns)
//
// These columns exist on observations, forecasts, and alert_runs:
// - event_id TEXT -> event.id
// - event_kind TEXT -> event.kind
// - event_source TEXT -> event.source
// - event_schema TEXT -> event.schema
// - event_emitted_at TIMESTAMPTZ -> event.emitted_at
// - event_effective_at TIMESTAMPTZ NULL -> event.effective_at
//
// Table contract
//
// 1. observations (PK: event_id)
//
// - event_id TEXT -> event.id
// - event_kind TEXT -> event.kind
// - event_source TEXT -> event.source
// - event_schema TEXT -> event.schema
// - event_emitted_at TIMESTAMPTZ -> event.emitted_at
// - event_effective_at TIMESTAMPTZ NULL -> event.effective_at
// - station_id TEXT NULL -> payload.stationId
// - station_name TEXT NULL -> payload.stationName
// - observed_at TIMESTAMPTZ -> payload.timestamp
// - condition_code INTEGER -> payload.conditionCode
// - is_day BOOLEAN NULL -> payload.isDay
// - text_description TEXT NULL -> payload.textDescription
// - temperature_c DOUBLE PRECISION NULL -> payload.temperatureC
// - dewpoint_c DOUBLE PRECISION NULL -> payload.dewpointC
// - wind_direction_degrees DOUBLE PRECISION NULL -> payload.windDirectionDegrees
// - wind_speed_kmh DOUBLE PRECISION NULL -> payload.windSpeedKmh
// - wind_gust_kmh DOUBLE PRECISION NULL -> payload.windGustKmh
// - barometric_pressure_pa DOUBLE PRECISION NULL -> payload.barometricPressurePa
// - visibility_meters DOUBLE PRECISION NULL -> payload.visibilityMeters
// - relative_humidity_percent DOUBLE PRECISION NULL -> payload.relativeHumidityPercent
// - apparent_temperature_c DOUBLE PRECISION NULL -> payload.apparentTemperatureC
//
// 2. observation_present_weather (PK: event_id, weather_index)
//
// - event_id TEXT -> observations.event_id / payload.presentWeather[i]
// - weather_index INTEGER -> i (array position in payload.presentWeather)
// - observed_at TIMESTAMPTZ -> payload.timestamp
// - raw_text TEXT NULL -> JSON-encoded payload.presentWeather[i].raw
//
// Note: raw_text stores compact JSON text. Consumers that need the original
// object should parse raw_text as JSON.
//
// 3. forecasts (PK: event_id)
//
// - event_id TEXT -> event.id
// - event_kind TEXT -> event.kind
// - event_source TEXT -> event.source
// - event_schema TEXT -> event.schema
// - event_emitted_at TIMESTAMPTZ -> event.emitted_at
// - event_effective_at TIMESTAMPTZ NULL -> event.effective_at
// - location_id TEXT NULL -> payload.locationId
// - location_name TEXT NULL -> payload.locationName
// - issued_at TIMESTAMPTZ -> payload.issuedAt
// - updated_at TIMESTAMPTZ NULL -> payload.updatedAt
// - product TEXT -> payload.product
// - latitude DOUBLE PRECISION NULL -> payload.latitude
// - longitude DOUBLE PRECISION NULL -> payload.longitude
// - elevation_meters DOUBLE PRECISION NULL -> payload.elevationMeters
// - period_count INTEGER -> len(payload.periods)
//
// 4. forecast_periods (PK: run_event_id, period_index)
//
// - run_event_id TEXT -> forecasts.event_id / payload.periods[i]
// - period_index INTEGER -> i (array position in payload.periods)
// - issued_at TIMESTAMPTZ -> payload.issuedAt (copied from parent)
// - start_time TIMESTAMPTZ -> payload.periods[i].startTime
// - end_time TIMESTAMPTZ -> payload.periods[i].endTime
// - name TEXT NULL -> payload.periods[i].name
// - is_day BOOLEAN NULL -> payload.periods[i].isDay
// - condition_code INTEGER -> payload.periods[i].conditionCode
// - text_description TEXT NULL -> payload.periods[i].textDescription
// - temperature_c DOUBLE PRECISION NULL -> payload.periods[i].temperatureC
// - temperature_c_min DOUBLE PRECISION NULL -> payload.periods[i].temperatureCMin
// - temperature_c_max DOUBLE PRECISION NULL -> payload.periods[i].temperatureCMax
// - dewpoint_c DOUBLE PRECISION NULL -> payload.periods[i].dewpointC
// - relative_humidity_percent DOUBLE PRECISION NULL -> payload.periods[i].relativeHumidityPercent
// - wind_direction_degrees DOUBLE PRECISION NULL -> payload.periods[i].windDirectionDegrees
// - wind_speed_kmh DOUBLE PRECISION NULL -> payload.periods[i].windSpeedKmh
// - wind_gust_kmh DOUBLE PRECISION NULL -> payload.periods[i].windGustKmh
// - barometric_pressure_pa DOUBLE PRECISION NULL -> payload.periods[i].barometricPressurePa
// - visibility_meters DOUBLE PRECISION NULL -> payload.periods[i].visibilityMeters
// - apparent_temperature_c DOUBLE PRECISION NULL -> payload.periods[i].apparentTemperatureC
// - cloud_cover_percent DOUBLE PRECISION NULL -> payload.periods[i].cloudCoverPercent
// - probability_of_precipitation_percent DOUBLE PRECISION NULL -> payload.periods[i].probabilityOfPrecipitationPercent
// - precipitation_amount_mm DOUBLE PRECISION NULL -> payload.periods[i].precipitationAmountMm
// - snowfall_depth_mm DOUBLE PRECISION NULL -> payload.periods[i].snowfallDepthMm
// - uv_index DOUBLE PRECISION NULL -> payload.periods[i].uvIndex
//
// 5. alert_runs (PK: event_id)
//
// - event_id TEXT -> event.id
// - event_kind TEXT -> event.kind
// - event_source TEXT -> event.source
// - event_schema TEXT -> event.schema
// - event_emitted_at TIMESTAMPTZ -> event.emitted_at
// - event_effective_at TIMESTAMPTZ NULL -> event.effective_at
// - location_id TEXT NULL -> payload.locationId
// - location_name TEXT NULL -> payload.locationName
// - as_of TIMESTAMPTZ -> payload.asOf
// - latitude DOUBLE PRECISION NULL -> payload.latitude
// - longitude DOUBLE PRECISION NULL -> payload.longitude
// - alert_count INTEGER -> len(payload.alerts)
//
// 6. alerts (PK: run_event_id, alert_index)
//
// - run_event_id TEXT -> alert_runs.event_id / payload.alerts[i]
// - alert_index INTEGER -> i (array position in payload.alerts)
// - as_of TIMESTAMPTZ -> payload.asOf (copied from parent)
// - alert_id TEXT -> payload.alerts[i].id
// - event TEXT NULL -> payload.alerts[i].event
// - headline TEXT NULL -> payload.alerts[i].headline
// - severity TEXT NULL -> payload.alerts[i].severity
// - urgency TEXT NULL -> payload.alerts[i].urgency
// - certainty TEXT NULL -> payload.alerts[i].certainty
// - status TEXT NULL -> payload.alerts[i].status
// - message_type TEXT NULL -> payload.alerts[i].messageType
// - category TEXT NULL -> payload.alerts[i].category
// - response TEXT NULL -> payload.alerts[i].response
// - description TEXT NULL -> payload.alerts[i].description
// - instruction TEXT NULL -> payload.alerts[i].instruction
// - sent TIMESTAMPTZ NULL -> payload.alerts[i].sent
// - effective TIMESTAMPTZ NULL -> payload.alerts[i].effective
// - onset TIMESTAMPTZ NULL -> payload.alerts[i].onset
// - expires TIMESTAMPTZ NULL -> payload.alerts[i].expires
// - area_description TEXT NULL -> payload.alerts[i].areaDescription
// - sender_name TEXT NULL -> payload.alerts[i].senderName
// - reference_count INTEGER -> len(payload.alerts[i].references)
//
// 7. alert_references (PK: run_event_id, alert_index, reference_index)
//
// - run_event_id TEXT -> alert_runs.event_id / payload.alerts[i].references[j]
// - alert_index INTEGER -> i (array position in payload.alerts)
// - reference_index INTEGER -> j (array position in payload.alerts[i].references)
// - as_of TIMESTAMPTZ -> payload.asOf (copied from parent)
// - id TEXT NULL -> payload.alerts[i].references[j].id
// - identifier TEXT NULL -> payload.alerts[i].references[j].identifier
// - sender TEXT NULL -> payload.alerts[i].references[j].sender
// - sent TIMESTAMPTZ NULL -> payload.alerts[i].references[j].sent
//
// Reconstructing canonical JSON payloads
//
// - WeatherObservation:
// read one row from observations, then join child rows by event_id ordered by
// weather_index to rebuild presentWeather arrays.
//
// - WeatherForecastRun:
// read one row from forecasts, then join forecast_periods by run_event_id
// ordered by period_index to rebuild periods.
//
// - WeatherAlertRun:
// read one row from alert_runs, join alerts by run_event_id ordered by
// alert_index, then join alert_references by (run_event_id, alert_index)
// ordered by reference_index to rebuild references per alert.
package postgres

View File

@@ -0,0 +1,385 @@
package postgres
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
fkevent "gitea.maximumdirect.net/ejr/feedkit/event"
fksinks "gitea.maximumdirect.net/ejr/feedkit/sinks"
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
func mapPostgresEvent(_ context.Context, e fkevent.Event) ([]fksinks.PostgresWrite, error) {
schema := strings.TrimSpace(e.Schema)
switch schema {
case standards.SchemaWeatherObservationV1:
return mapObservationEvent(e)
case standards.SchemaWeatherForecastV1:
return mapForecastEvent(e)
case standards.SchemaWeatherForecastDiscussionV1:
return mapForecastDiscussionEvent(e)
case standards.SchemaWeatherAlertV1:
return mapAlertEvent(e)
default:
return nil, nil
}
}
func mapObservationEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
obs, err := decodePayload[model.WeatherObservation](e.Payload)
if err != nil {
return nil, fmt.Errorf("decode observation payload: %w", err)
}
if obs.Timestamp.IsZero() {
return nil, fmt.Errorf("decode observation payload: timestamp is required")
}
observedAt := obs.Timestamp.UTC()
writes := make([]fksinks.PostgresWrite, 0, 1+len(obs.PresentWeather))
writes = append(writes, fksinks.PostgresWrite{
Table: tableObservations,
Values: map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
"station_id": nullableString(obs.StationID),
"station_name": nullableString(obs.StationName),
"observed_at": observedAt,
"condition_code": int(obs.ConditionCode),
"is_day": nullableBool(obs.IsDay),
"text_description": nullableString(obs.TextDescription),
"temperature_c": nullableFloat64(obs.TemperatureC),
"dewpoint_c": nullableFloat64(obs.DewpointC),
"wind_direction_degrees": nullableFloat64(obs.WindDirectionDegrees),
"wind_speed_kmh": nullableFloat64(obs.WindSpeedKmh),
"wind_gust_kmh": nullableFloat64(obs.WindGustKmh),
"barometric_pressure_pa": nullableFloat64(obs.BarometricPressurePa),
"visibility_meters": nullableFloat64(obs.VisibilityMeters),
"relative_humidity_percent": nullableFloat64(obs.RelativeHumidityPercent),
"apparent_temperature_c": nullableFloat64(obs.ApparentTemperatureC),
},
})
for i, pw := range obs.PresentWeather {
rawText, err := compactJSONText(pw.Raw)
if err != nil {
return nil, fmt.Errorf("observation presentWeather[%d].raw: %w", i, err)
}
writes = append(writes, fksinks.PostgresWrite{
Table: tableObservationPresentWeather,
Values: map[string]any{
"event_id": e.ID,
"weather_index": i,
"observed_at": observedAt,
"raw_text": rawText,
},
})
}
return writes, nil
}
func mapForecastEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
run, err := decodePayload[model.WeatherForecastRun](e.Payload)
if err != nil {
return nil, fmt.Errorf("decode forecast payload: %w", err)
}
if run.IssuedAt.IsZero() {
return nil, fmt.Errorf("decode forecast payload: issuedAt is required")
}
if strings.TrimSpace(string(run.Product)) == "" {
return nil, fmt.Errorf("decode forecast payload: product is required")
}
issuedAt := run.IssuedAt.UTC()
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.Periods))
writes = append(writes, fksinks.PostgresWrite{
Table: tableForecasts,
Values: map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
"location_id": nullableString(run.LocationID),
"location_name": nullableString(run.LocationName),
"issued_at": issuedAt,
"updated_at": nullableTime(run.UpdatedAt),
"product": string(run.Product),
"latitude": nullableFloat64(run.Latitude),
"longitude": nullableFloat64(run.Longitude),
"elevation_meters": nullableFloat64(run.ElevationMeters),
"period_count": len(run.Periods),
},
})
for i, p := range run.Periods {
if p.StartTime.IsZero() || p.EndTime.IsZero() {
return nil, fmt.Errorf("decode forecast payload: periods[%d] startTime/endTime are required", i)
}
writes = append(writes, fksinks.PostgresWrite{
Table: tableForecastPeriods,
Values: map[string]any{
"run_event_id": e.ID,
"period_index": i,
"issued_at": issuedAt,
"start_time": p.StartTime.UTC(),
"end_time": p.EndTime.UTC(),
"name": nullableString(p.Name),
"is_day": nullableBool(p.IsDay),
"condition_code": int(p.ConditionCode),
"text_description": nullableString(p.TextDescription),
"temperature_c": nullableFloat64(p.TemperatureC),
"temperature_c_min": nullableFloat64(p.TemperatureCMin),
"temperature_c_max": nullableFloat64(p.TemperatureCMax),
"dewpoint_c": nullableFloat64(p.DewpointC),
"relative_humidity_percent": nullableFloat64(p.RelativeHumidityPercent),
"wind_direction_degrees": nullableFloat64(p.WindDirectionDegrees),
"wind_speed_kmh": nullableFloat64(p.WindSpeedKmh),
"wind_gust_kmh": nullableFloat64(p.WindGustKmh),
"barometric_pressure_pa": nullableFloat64(p.BarometricPressurePa),
"visibility_meters": nullableFloat64(p.VisibilityMeters),
"apparent_temperature_c": nullableFloat64(p.ApparentTemperatureC),
"cloud_cover_percent": nullableFloat64(p.CloudCoverPercent),
"probability_of_precipitation_percent": nullableFloat64(p.ProbabilityOfPrecipitationPercent),
"precipitation_amount_mm": nullableFloat64(p.PrecipitationAmountMm),
"snowfall_depth_mm": nullableFloat64(p.SnowfallDepthMM),
"uv_index": nullableFloat64(p.UVIndex),
},
})
}
return writes, nil
}
func mapForecastDiscussionEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
run, err := decodePayload[model.WeatherForecastDiscussion](e.Payload)
if err != nil {
return nil, fmt.Errorf("decode forecast discussion payload: %w", err)
}
if run.IssuedAt.IsZero() {
return nil, fmt.Errorf("decode forecast discussion payload: issuedAt is required")
}
if strings.TrimSpace(string(run.Product)) == "" {
return nil, fmt.Errorf("decode forecast discussion payload: product is required")
}
issuedAt := run.IssuedAt.UTC()
shortTermQualifier, shortTermIssuedAt, shortTermText := nullableDiscussionSection(run.ShortTerm)
longTermQualifier, longTermIssuedAt, longTermText := nullableDiscussionSection(run.LongTerm)
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.KeyMessages))
writes = append(writes, fksinks.PostgresWrite{
Table: tableForecastDiscussions,
Values: map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
"office_id": nullableString(run.OfficeID),
"office_name": nullableString(run.OfficeName),
"issued_at": issuedAt,
"updated_at": nullableTime(run.UpdatedAt),
"product": string(run.Product),
"short_term_qualifier": shortTermQualifier,
"short_term_issued_at": shortTermIssuedAt,
"short_term_text": shortTermText,
"long_term_qualifier": longTermQualifier,
"long_term_issued_at": longTermIssuedAt,
"long_term_text": longTermText,
"key_message_count": len(run.KeyMessages),
},
})
for i, msg := range run.KeyMessages {
writes = append(writes, fksinks.PostgresWrite{
Table: tableForecastDiscussionKeyMessages,
Values: map[string]any{
"run_event_id": e.ID,
"message_index": i,
"issued_at": issuedAt,
"message_text": nullableString(msg),
},
})
}
return writes, nil
}
func mapAlertEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
run, err := decodePayload[model.WeatherAlertRun](e.Payload)
if err != nil {
return nil, fmt.Errorf("decode alert payload: %w", err)
}
if run.AsOf.IsZero() {
return nil, fmt.Errorf("decode alert payload: asOf is required")
}
asOf := run.AsOf.UTC()
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.Alerts)+countAlertReferences(run.Alerts))
writes = append(writes, fksinks.PostgresWrite{
Table: tableAlertRuns,
Values: map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
"location_id": nullableString(run.LocationID),
"location_name": nullableString(run.LocationName),
"as_of": asOf,
"latitude": nullableFloat64(run.Latitude),
"longitude": nullableFloat64(run.Longitude),
"alert_count": len(run.Alerts),
},
})
for i, a := range run.Alerts {
if strings.TrimSpace(a.ID) == "" {
return nil, fmt.Errorf("decode alert payload: alerts[%d].id is required", i)
}
writes = append(writes, fksinks.PostgresWrite{
Table: tableAlerts,
Values: map[string]any{
"run_event_id": e.ID,
"alert_index": i,
"as_of": asOf,
"alert_id": a.ID,
"event": nullableString(a.Event),
"headline": nullableString(a.Headline),
"severity": nullableString(a.Severity),
"urgency": nullableString(a.Urgency),
"certainty": nullableString(a.Certainty),
"status": nullableString(a.Status),
"message_type": nullableString(a.MessageType),
"category": nullableString(a.Category),
"response": nullableString(a.Response),
"description": nullableString(a.Description),
"instruction": nullableString(a.Instruction),
"sent": nullableTime(a.Sent),
"effective": nullableTime(a.Effective),
"onset": nullableTime(a.Onset),
"expires": nullableTime(a.Expires),
"area_description": nullableString(a.AreaDescription),
"sender_name": nullableString(a.SenderName),
"reference_count": len(a.References),
},
})
for j, ref := range a.References {
writes = append(writes, fksinks.PostgresWrite{
Table: tableAlertReferences,
Values: map[string]any{
"run_event_id": e.ID,
"alert_index": i,
"reference_index": j,
"as_of": asOf,
"id": nullableString(ref.ID),
"identifier": nullableString(ref.Identifier),
"sender": nullableString(ref.Sender),
"sent": nullableTime(ref.Sent),
},
})
}
}
return writes, nil
}
func decodePayload[T any](payload any) (T, error) {
var out T
if payload == nil {
return out, fmt.Errorf("payload is nil")
}
if typed, ok := payload.(T); ok {
return typed, nil
}
if ptr, ok := payload.(*T); ok {
if ptr == nil {
return out, fmt.Errorf("payload pointer is nil")
}
return *ptr, nil
}
b, err := json.Marshal(payload)
if err != nil {
return out, fmt.Errorf("marshal payload: %w", err)
}
if err := json.Unmarshal(b, &out); err != nil {
return out, fmt.Errorf("unmarshal payload: %w", err)
}
return out, nil
}
func nullableDiscussionSection(section *model.WeatherForecastDiscussionSection) (any, any, any) {
if section == nil {
return nil, nil, nil
}
return nullableString(section.Qualifier), nullableTime(section.IssuedAt), nullableString(section.Text)
}
func countAlertReferences(alerts []model.WeatherAlert) int {
total := 0
for _, a := range alerts {
total += len(a.References)
}
return total
}
func nullableString(s string) any {
if strings.TrimSpace(s) == "" {
return nil
}
return s
}
func nullableFloat64(v *float64) any {
if v == nil {
return nil
}
return *v
}
func nullableBool(v *bool) any {
if v == nil {
return nil
}
return *v
}
func nullableTime(v *time.Time) any {
if v == nil || v.IsZero() {
return nil
}
return v.UTC()
}
func compactJSONText(v any) (any, error) {
if v == nil {
return nil, nil
}
b, err := json.Marshal(v)
if err != nil {
return nil, err
}
if string(b) == "null" {
return nil, nil
}
return string(b), nil
}

View File

@@ -0,0 +1,301 @@
package postgres
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
fkevent "gitea.maximumdirect.net/ejr/feedkit/event"
fksinks "gitea.maximumdirect.net/ejr/feedkit/sinks"
"gitea.maximumdirect.net/ejr/weatherfeeder/model"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
func TestMapPostgresEventObservationStructPayload(t *testing.T) {
isDay := true
temp := 21.5
obs := model.WeatherObservation{
StationID: "KSTL",
StationName: "St. Louis",
Timestamp: time.Date(2026, 3, 16, 19, 0, 0, 0, time.UTC),
ConditionCode: model.WMOCode(1),
IsDay: &isDay,
TextDescription: "few clouds",
TemperatureC: &temp,
PresentWeather: []model.PresentWeather{{Raw: map[string]any{"a": 1, "b": "x"}}},
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherObservationV1, "observation", obs))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
if len(writes) != 2 {
t.Fatalf("mapPostgresEvent() writes len = %d, want 2", len(writes))
}
if writes[0].Table != tableObservations {
t.Fatalf("writes[0].Table = %q, want %q", writes[0].Table, tableObservations)
}
if got := writes[0].Values["station_id"]; got != "KSTL" {
t.Fatalf("observations station_id = %#v, want KSTL", got)
}
if writes[1].Table != tableObservationPresentWeather {
t.Fatalf("writes[1].Table = %q, want %q", writes[1].Table, tableObservationPresentWeather)
}
if got := writes[1].Values["raw_text"]; got != `{"a":1,"b":"x"}` {
t.Fatalf("present_weather raw_text = %#v, want compact JSON", got)
}
assertAllWritesIncludeAllColumns(t, writes)
}
func TestMapPostgresEventForecastStructPayload(t *testing.T) {
isDay := true
temp := 10.5
run := model.WeatherForecastRun{
LocationID: "LOC-1",
LocationName: "St. Louis",
IssuedAt: time.Date(2026, 3, 16, 18, 0, 0, 0, time.UTC),
Product: model.ForecastProductHourly,
Periods: []model.WeatherForecastPeriod{
{
StartTime: time.Date(2026, 3, 16, 19, 0, 0, 0, time.UTC),
EndTime: time.Date(2026, 3, 16, 20, 0, 0, 0, time.UTC),
IsDay: &isDay,
ConditionCode: model.WMOCode(2),
TemperatureC: &temp,
},
{
StartTime: time.Date(2026, 3, 16, 20, 0, 0, 0, time.UTC),
EndTime: time.Date(2026, 3, 16, 21, 0, 0, 0, time.UTC),
ConditionCode: model.WMOCode(3),
},
},
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, "forecast", run))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
if len(writes) != 3 {
t.Fatalf("mapPostgresEvent() writes len = %d, want 3", len(writes))
}
if writes[0].Table != tableForecasts {
t.Fatalf("writes[0].Table = %q, want %q", writes[0].Table, tableForecasts)
}
if got := writes[0].Values["period_count"]; got != 2 {
t.Fatalf("forecasts period_count = %#v, want 2", got)
}
if writes[1].Table != tableForecastPeriods || writes[2].Table != tableForecastPeriods {
t.Fatalf("forecast period writes not in expected order")
}
if got := writes[1].Values["period_index"]; got != 0 {
t.Fatalf("first period index = %#v, want 0", got)
}
assertAllWritesIncludeAllColumns(t, writes)
}
func TestMapPostgresEventAlertStructPayload(t *testing.T) {
sent := time.Date(2026, 3, 16, 17, 0, 0, 0, time.UTC)
run := model.WeatherAlertRun{
AsOf: time.Date(2026, 3, 16, 18, 0, 0, 0, time.UTC),
Alerts: []model.WeatherAlert{
{
ID: "urn:alert:1",
Headline: "Winter Weather Advisory",
Severity: "Moderate",
References: []model.AlertReference{
{ID: "urn:ref:1", Sent: &sent},
{Identifier: "ref-two"},
},
},
{
ID: "urn:alert:2",
Headline: "Second alert",
},
},
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherAlertV1, "alert", run))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
if len(writes) != 5 {
t.Fatalf("mapPostgresEvent() writes len = %d, want 5", len(writes))
}
counts := map[string]int{}
for _, w := range writes {
counts[w.Table]++
}
if counts[tableAlertRuns] != 1 || counts[tableAlerts] != 2 || counts[tableAlertReferences] != 2 {
t.Fatalf("unexpected table write counts: %#v", counts)
}
firstAlert, ok := firstWriteForTable(writes, tableAlerts)
if !ok {
t.Fatalf("missing alerts write")
}
if got := firstAlert.Values["reference_count"]; got != 2 {
t.Fatalf("alerts reference_count = %#v, want 2", got)
}
assertAllWritesIncludeAllColumns(t, writes)
}
func TestMapPostgresEventForecastDiscussionStructPayload(t *testing.T) {
updatedAt := time.Date(2026, 3, 28, 20, 29, 47, 0, time.UTC)
shortIssuedAt := time.Date(2026, 3, 28, 19, 19, 0, 0, time.UTC)
run := model.WeatherForecastDiscussion{
OfficeID: "LSX",
OfficeName: "National Weather Service Saint Louis MO",
Product: model.ForecastDiscussionProductAFD,
IssuedAt: time.Date(2026, 3, 28, 19, 24, 0, 0, time.UTC),
UpdatedAt: &updatedAt,
KeyMessages: []string{"msg one", "msg two"},
ShortTerm: &model.WeatherForecastDiscussionSection{Qualifier: "(Tonight)", IssuedAt: &shortIssuedAt, Text: "Short term text"},
LongTerm: &model.WeatherForecastDiscussionSection{Text: "Long term text"},
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, "forecast_discussion", run))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
if len(writes) != 3 {
t.Fatalf("mapPostgresEvent() writes len = %d, want 3", len(writes))
}
if writes[0].Table != tableForecastDiscussions {
t.Fatalf("writes[0].Table = %q, want %q", writes[0].Table, tableForecastDiscussions)
}
if got := writes[0].Values["key_message_count"]; got != 2 {
t.Fatalf("forecast_discussions key_message_count = %#v, want 2", got)
}
if got := writes[0].Values["short_term_qualifier"]; got != "(Tonight)" {
t.Fatalf("forecast_discussions short_term_qualifier = %#v, want (Tonight)", got)
}
if got := writes[0].Values["long_term_issued_at"]; got != nil {
t.Fatalf("forecast_discussions long_term_issued_at = %#v, want nil", got)
}
if writes[1].Table != tableForecastDiscussionKeyMessages || writes[2].Table != tableForecastDiscussionKeyMessages {
t.Fatalf("forecast discussion key message writes not in expected order")
}
if got := writes[2].Values["message_index"]; got != 1 {
t.Fatalf("second key message index = %#v, want 1", got)
}
assertAllWritesIncludeAllColumns(t, writes)
}
func TestMapPostgresEventMapPayload(t *testing.T) {
run := model.WeatherForecastRun{
IssuedAt: time.Date(2026, 3, 16, 18, 0, 0, 0, time.UTC),
Product: model.ForecastProductHourly,
Periods: []model.WeatherForecastPeriod{
{
StartTime: time.Date(2026, 3, 16, 19, 0, 0, 0, time.UTC),
EndTime: time.Date(2026, 3, 16, 20, 0, 0, 0, time.UTC),
ConditionCode: model.WMOCode(2),
},
},
}
b, err := json.Marshal(run)
if err != nil {
t.Fatalf("json.Marshal() error = %v", err)
}
var payload map[string]any
if err := json.Unmarshal(b, &payload); err != nil {
t.Fatalf("json.Unmarshal() error = %v", err)
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, "forecast", payload))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
if len(writes) != 2 {
t.Fatalf("mapPostgresEvent() writes len = %d, want 2", len(writes))
}
if writes[0].Table != tableForecasts {
t.Fatalf("writes[0].Table = %q, want %q", writes[0].Table, tableForecasts)
}
assertAllWritesIncludeAllColumns(t, writes)
}
func TestMapPostgresEventUnknownSchemaNoOp(t *testing.T) {
writes, err := mapPostgresEvent(context.Background(), testEvent("weather.unknown.v1", "observation", map[string]any{"x": 1}))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
if len(writes) != 0 {
t.Fatalf("mapPostgresEvent() writes len = %d, want 0", len(writes))
}
}
func TestMapPostgresEventMalformedPayload(t *testing.T) {
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, "forecast", "bad"))
if err == nil {
t.Fatalf("mapPostgresEvent() expected error for malformed payload")
}
if !strings.Contains(err.Error(), "decode forecast payload") {
t.Fatalf("error = %q, want decode forecast payload context", err)
}
}
func TestMapPostgresEventForecastDiscussionMalformedPayload(t *testing.T) {
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, "forecast_discussion", "bad"))
if err == nil {
t.Fatalf("mapPostgresEvent() expected error for malformed payload")
}
if !strings.Contains(err.Error(), "decode forecast discussion payload") {
t.Fatalf("error = %q, want decode forecast discussion payload context", err)
}
}
func testEvent(schema string, kind fkevent.Kind, payload any) fkevent.Event {
effectiveAt := time.Date(2026, 3, 16, 18, 30, 0, 0, time.UTC)
return fkevent.Event{
ID: "evt-1",
Kind: kind,
Source: "test-source",
Schema: schema,
EmittedAt: time.Date(2026, 3, 16, 18, 31, 0, 0, time.UTC),
EffectiveAt: &effectiveAt,
Payload: payload,
}
}
func firstWriteForTable(writes []fksinks.PostgresWrite, table string) (fksinks.PostgresWrite, bool) {
for _, w := range writes {
if w.Table == table {
return w, true
}
}
return fksinks.PostgresWrite{}, false
}
func assertAllWritesIncludeAllColumns(t *testing.T, writes []fksinks.PostgresWrite) {
t.Helper()
colCounts := tableColumnCounts()
for i, w := range writes {
expectedCount, ok := colCounts[w.Table]
if !ok {
t.Fatalf("writes[%d] references unknown table %q", i, w.Table)
}
if len(w.Values) != expectedCount {
t.Fatalf("writes[%d] table=%q has %d values, want %d", i, w.Table, len(w.Values), expectedCount)
}
}
}
func tableColumnCounts() map[string]int {
s := PostgresSchema()
m := make(map[string]int, len(s.Tables))
for _, tbl := range s.Tables {
m[tbl.Name] = len(tbl.Columns)
}
return m
}

View File

@@ -0,0 +1,256 @@
package postgres
import (
fksinks "gitea.maximumdirect.net/ejr/feedkit/sinks"
)
const (
tableObservations = "observations"
tableObservationPresentWeather = "observation_present_weather"
tableForecasts = "forecasts"
tableForecastPeriods = "forecast_periods"
tableForecastDiscussions = "forecast_discussions"
tableForecastDiscussionKeyMessages = "forecast_discussion_key_messages"
tableAlertRuns = "alert_runs"
tableAlerts = "alerts"
tableAlertReferences = "alert_references"
)
// PostgresSchema returns weatherfeeder's Postgres schema definition.
func PostgresSchema() fksinks.PostgresSchema {
return fksinks.PostgresSchema{
Tables: []fksinks.PostgresTable{
{
Name: tableObservations,
Columns: []fksinks.PostgresColumn{
{Name: "event_id", Type: "TEXT", Nullable: false},
{Name: "event_kind", Type: "TEXT", Nullable: false},
{Name: "event_source", Type: "TEXT", Nullable: false},
{Name: "event_schema", Type: "TEXT", Nullable: false},
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "station_id", Type: "TEXT", Nullable: true},
{Name: "station_name", Type: "TEXT", Nullable: true},
{Name: "observed_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "condition_code", Type: "INTEGER", Nullable: false},
{Name: "is_day", Type: "BOOLEAN", Nullable: true},
{Name: "text_description", Type: "TEXT", Nullable: true},
{Name: "temperature_c", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "dewpoint_c", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "wind_direction_degrees", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "wind_speed_kmh", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "wind_gust_kmh", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "barometric_pressure_pa", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "visibility_meters", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "relative_humidity_percent", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "apparent_temperature_c", Type: "DOUBLE PRECISION", Nullable: true},
},
PrimaryKey: []string{"event_id"},
PruneColumn: "observed_at",
Indexes: []fksinks.PostgresIndex{
{Name: "idx_wf_obs_station_observed_at", Columns: []string{"station_id", "observed_at"}},
{Name: "idx_wf_obs_observed_at", Columns: []string{"observed_at"}},
{Name: "idx_wf_obs_condition_code", Columns: []string{"condition_code"}},
},
},
{
Name: tableObservationPresentWeather,
Columns: []fksinks.PostgresColumn{
{Name: "event_id", Type: "TEXT REFERENCES observations(event_id) ON DELETE CASCADE", Nullable: false},
{Name: "weather_index", Type: "INTEGER", Nullable: false},
{Name: "observed_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "raw_text", Type: "TEXT", Nullable: true},
},
PrimaryKey: []string{"event_id", "weather_index"},
PruneColumn: "observed_at",
Indexes: []fksinks.PostgresIndex{
{Name: "idx_wf_obs_present_observed_at", Columns: []string{"observed_at"}},
},
},
{
Name: tableForecasts,
Columns: []fksinks.PostgresColumn{
{Name: "event_id", Type: "TEXT", Nullable: false},
{Name: "event_kind", Type: "TEXT", Nullable: false},
{Name: "event_source", Type: "TEXT", Nullable: false},
{Name: "event_schema", Type: "TEXT", Nullable: false},
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "location_id", Type: "TEXT", Nullable: true},
{Name: "location_name", Type: "TEXT", Nullable: true},
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "updated_at", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "product", Type: "TEXT", Nullable: false},
{Name: "latitude", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "longitude", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "elevation_meters", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "period_count", Type: "INTEGER", Nullable: false},
},
PrimaryKey: []string{"event_id"},
PruneColumn: "issued_at",
Indexes: []fksinks.PostgresIndex{
{Name: "idx_wf_fc_location_product_issued_at", Columns: []string{"location_id", "product", "issued_at"}},
{Name: "idx_wf_fc_issued_at", Columns: []string{"issued_at"}},
{Name: "idx_wf_fc_product_issued_at", Columns: []string{"product", "issued_at"}},
},
},
{
Name: tableForecastPeriods,
Columns: []fksinks.PostgresColumn{
{Name: "run_event_id", Type: "TEXT REFERENCES forecasts(event_id) ON DELETE CASCADE", Nullable: false},
{Name: "period_index", Type: "INTEGER", Nullable: false},
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "start_time", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "end_time", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "name", Type: "TEXT", Nullable: true},
{Name: "is_day", Type: "BOOLEAN", Nullable: true},
{Name: "condition_code", Type: "INTEGER", Nullable: false},
{Name: "text_description", Type: "TEXT", Nullable: true},
{Name: "temperature_c", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "temperature_c_min", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "temperature_c_max", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "dewpoint_c", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "relative_humidity_percent", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "wind_direction_degrees", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "wind_speed_kmh", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "wind_gust_kmh", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "barometric_pressure_pa", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "visibility_meters", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "apparent_temperature_c", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "cloud_cover_percent", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "probability_of_precipitation_percent", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "precipitation_amount_mm", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "snowfall_depth_mm", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "uv_index", Type: "DOUBLE PRECISION", Nullable: true},
},
PrimaryKey: []string{"run_event_id", "period_index"},
PruneColumn: "issued_at",
Indexes: []fksinks.PostgresIndex{
{Name: "idx_wf_fc_period_start_time", Columns: []string{"start_time"}},
{Name: "idx_wf_fc_period_end_time", Columns: []string{"end_time"}},
{Name: "idx_wf_fc_period_run_start", Columns: []string{"run_event_id", "start_time"}},
},
},
{
Name: tableForecastDiscussions,
Columns: []fksinks.PostgresColumn{
{Name: "event_id", Type: "TEXT", Nullable: false},
{Name: "event_kind", Type: "TEXT", Nullable: false},
{Name: "event_source", Type: "TEXT", Nullable: false},
{Name: "event_schema", Type: "TEXT", Nullable: false},
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "office_id", Type: "TEXT", Nullable: true},
{Name: "office_name", Type: "TEXT", Nullable: true},
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "updated_at", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "product", Type: "TEXT", Nullable: false},
{Name: "short_term_qualifier", Type: "TEXT", Nullable: true},
{Name: "short_term_issued_at", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "short_term_text", Type: "TEXT", Nullable: true},
{Name: "long_term_qualifier", Type: "TEXT", Nullable: true},
{Name: "long_term_issued_at", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "long_term_text", Type: "TEXT", Nullable: true},
{Name: "key_message_count", Type: "INTEGER", Nullable: false},
},
PrimaryKey: []string{"event_id"},
PruneColumn: "issued_at",
Indexes: []fksinks.PostgresIndex{
{Name: "idx_wf_discussion_office_product_issued_at", Columns: []string{"office_id", "product", "issued_at"}},
{Name: "idx_wf_discussion_issued_at", Columns: []string{"issued_at"}},
},
},
{
Name: tableForecastDiscussionKeyMessages,
Columns: []fksinks.PostgresColumn{
{Name: "run_event_id", Type: "TEXT REFERENCES forecast_discussions(event_id) ON DELETE CASCADE", Nullable: false},
{Name: "message_index", Type: "INTEGER", Nullable: false},
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "message_text", Type: "TEXT", Nullable: true},
},
PrimaryKey: []string{"run_event_id", "message_index"},
PruneColumn: "issued_at",
Indexes: []fksinks.PostgresIndex{
{Name: "idx_wf_discussion_message_issued_at", Columns: []string{"issued_at"}},
},
},
{
Name: tableAlertRuns,
Columns: []fksinks.PostgresColumn{
{Name: "event_id", Type: "TEXT", Nullable: false},
{Name: "event_kind", Type: "TEXT", Nullable: false},
{Name: "event_source", Type: "TEXT", Nullable: false},
{Name: "event_schema", Type: "TEXT", Nullable: false},
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "location_id", Type: "TEXT", Nullable: true},
{Name: "location_name", Type: "TEXT", Nullable: true},
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "latitude", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "longitude", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "alert_count", Type: "INTEGER", Nullable: false},
},
PrimaryKey: []string{"event_id"},
PruneColumn: "as_of",
Indexes: []fksinks.PostgresIndex{
{Name: "idx_wf_alert_run_location_as_of", Columns: []string{"location_id", "as_of"}},
{Name: "idx_wf_alert_run_as_of", Columns: []string{"as_of"}},
},
},
{
Name: tableAlerts,
Columns: []fksinks.PostgresColumn{
{Name: "run_event_id", Type: "TEXT REFERENCES alert_runs(event_id) ON DELETE CASCADE", Nullable: false},
{Name: "alert_index", Type: "INTEGER", Nullable: false},
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "alert_id", Type: "TEXT", Nullable: false},
{Name: "event", Type: "TEXT", Nullable: true},
{Name: "headline", Type: "TEXT", Nullable: true},
{Name: "severity", Type: "TEXT", Nullable: true},
{Name: "urgency", Type: "TEXT", Nullable: true},
{Name: "certainty", Type: "TEXT", Nullable: true},
{Name: "status", Type: "TEXT", Nullable: true},
{Name: "message_type", Type: "TEXT", Nullable: true},
{Name: "category", Type: "TEXT", Nullable: true},
{Name: "response", Type: "TEXT", Nullable: true},
{Name: "description", Type: "TEXT", Nullable: true},
{Name: "instruction", Type: "TEXT", Nullable: true},
{Name: "sent", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "effective", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "onset", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "expires", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "area_description", Type: "TEXT", Nullable: true},
{Name: "sender_name", Type: "TEXT", Nullable: true},
{Name: "reference_count", Type: "INTEGER", Nullable: false},
},
PrimaryKey: []string{"run_event_id", "alert_index"},
PruneColumn: "as_of",
Indexes: []fksinks.PostgresIndex{
{Name: "idx_wf_alerts_alert_id", Columns: []string{"alert_id"}},
{Name: "idx_wf_alerts_severity_expires", Columns: []string{"severity", "expires"}},
{Name: "idx_wf_alerts_as_of", Columns: []string{"as_of"}},
},
},
{
Name: tableAlertReferences,
Columns: []fksinks.PostgresColumn{
{Name: "run_event_id", Type: "TEXT REFERENCES alert_runs(event_id) ON DELETE CASCADE", Nullable: false},
{Name: "alert_index", Type: "INTEGER", Nullable: false},
{Name: "reference_index", Type: "INTEGER", Nullable: false},
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "id", Type: "TEXT", Nullable: true},
{Name: "identifier", Type: "TEXT", Nullable: true},
{Name: "sender", Type: "TEXT", Nullable: true},
{Name: "sent", Type: "TIMESTAMPTZ", Nullable: true},
},
PrimaryKey: []string{"run_event_id", "alert_index", "reference_index"},
PruneColumn: "as_of",
Indexes: []fksinks.PostgresIndex{
{Name: "idx_wf_alert_refs_as_of", Columns: []string{"as_of"}},
{Name: "idx_wf_alert_refs_sent", Columns: []string{"sent"}},
},
},
},
MapEvent: mapPostgresEvent,
}
}

View File

@@ -0,0 +1,42 @@
package postgres
import "testing"
func TestWeatherPostgresSchemaShape(t *testing.T) {
s := PostgresSchema()
if s.MapEvent == nil {
t.Fatalf("PostgresSchema().MapEvent is nil")
}
wantTables := map[string]bool{
tableObservations: true,
tableObservationPresentWeather: true,
tableForecasts: true,
tableForecastPeriods: true,
tableForecastDiscussions: true,
tableForecastDiscussionKeyMessages: true,
tableAlertRuns: true,
tableAlerts: true,
tableAlertReferences: true,
}
if len(s.Tables) != len(wantTables) {
t.Fatalf("PostgresSchema().Tables len = %d, want %d", len(s.Tables), len(wantTables))
}
seenIndexes := map[string]bool{}
for _, tbl := range s.Tables {
if !wantTables[tbl.Name] {
t.Fatalf("unexpected table %q in schema", tbl.Name)
}
if tbl.PruneColumn == "" {
t.Fatalf("table %q missing prune column", tbl.Name)
}
for _, idx := range tbl.Indexes {
if seenIndexes[idx.Name] {
t.Fatalf("duplicate index name %q", idx.Name)
}
seenIndexes[idx.Name] = true
}
}
}

View File

@@ -1,8 +1,6 @@
package sources package sources
import ( import (
"fmt"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/nws" "gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/nws"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/openmeteo" "gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/openmeteo"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/openweather" "gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/openweather"
@@ -11,46 +9,33 @@ import (
fksource "gitea.maximumdirect.net/ejr/feedkit/sources" fksource "gitea.maximumdirect.net/ejr/feedkit/sources"
) )
type pollDriverRegistration struct {
driver string
factory func(config.SourceConfig) (fksource.PollSource, error)
}
var pollDriverRegistrations = []pollDriverRegistration{
{driver: "nws_observation", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewObservationSource(cfg) }},
{driver: "nws_alerts", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewAlertsSource(cfg) }},
{driver: "nws_forecast_hourly", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewHourlyForecastSource(cfg) }},
{driver: "nws_forecast_narrative", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewNarrativeForecastSource(cfg) }},
{driver: "nws_forecast_discussion", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
return nws.NewForecastDiscussionSource(cfg)
}},
{driver: "openmeteo_observation", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewObservationSource(cfg) }},
{driver: "openmeteo_forecast", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewForecastSource(cfg) }},
{driver: "openweather_observation", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
return openweather.NewObservationSource(cfg)
}},
}
// RegisterBuiltins registers the source drivers that ship with this binary. // RegisterBuiltins registers the source drivers that ship with this binary.
// Keeping this in one place makes main.go very readable. // Keeping this in one place makes main.go very readable.
func RegisterBuiltins(r *fksource.Registry) { func RegisterBuiltins(r *fksource.Registry) {
// NWS drivers for _, reg := range pollDriverRegistrations {
r.Register("nws_observation", func(cfg config.SourceConfig) (fksource.Source, error) { reg := reg
return nws.NewObservationSource(cfg) r.RegisterPoll(reg.driver, func(cfg config.SourceConfig) (fksource.PollSource, error) {
}) return reg.factory(cfg)
r.Register("nws_alerts", func(cfg config.SourceConfig) (fksource.Source, error) { })
return nws.NewAlertsSource(cfg)
})
r.Register("nws_forecast", func(cfg config.SourceConfig) (fksource.Source, error) {
return nws.NewForecastSource(cfg)
})
// Open-Meteo drivers
r.Register("openmeteo_observation", func(cfg config.SourceConfig) (fksource.Source, error) {
return openmeteo.NewObservationSource(cfg)
})
// OpenWeatherMap drivers
r.Register("openweather_observation", func(cfg config.SourceConfig) (fksource.Source, error) {
return openweather.NewObservationSource(cfg)
})
}
// Optional: centralize some common config checks used by multiple drivers.
//
// NOTE: feedkit/config.SourceConfig intentionally keeps driver-specific options
// inside cfg.Params, so drivers can evolve independently without feedkit
// importing domain config packages.
func RequireURL(cfg config.SourceConfig) error {
if cfg.Params == nil {
return fmt.Errorf("source %q: params.url is required", cfg.Name)
} }
// Canonical key is "url". We also accept "URL" as a convenience.
url, ok := cfg.ParamString("url", "URL")
if !ok {
return fmt.Errorf("source %q: params.url is required", cfg.Name)
}
_ = url // (optional) return it if you want this helper to provide the value
return nil
} }

View File

@@ -0,0 +1,103 @@
package sources
import (
"strings"
"testing"
"gitea.maximumdirect.net/ejr/feedkit/config"
fksource "gitea.maximumdirect.net/ejr/feedkit/sources"
)
func TestRegisterBuiltinsRegistersNWSHourlyForecastDriver(t *testing.T) {
reg := fksource.NewRegistry()
RegisterBuiltins(reg)
in, err := reg.BuildInput(sourceConfigForDriver("nws_forecast_hourly"))
if err != nil {
t.Fatalf("BuildInput(nws_forecast_hourly) error = %v", err)
}
if _, ok := in.(fksource.PollSource); !ok {
t.Fatalf("BuildInput(nws_forecast_hourly) type = %T, want PollSource", in)
}
}
func TestRegisterBuiltinsRegistersNWSNarrativeForecastDriver(t *testing.T) {
reg := fksource.NewRegistry()
RegisterBuiltins(reg)
in, err := reg.BuildInput(sourceConfigForDriver("nws_forecast_narrative"))
if err != nil {
t.Fatalf("BuildInput(nws_forecast_narrative) error = %v", err)
}
if _, ok := in.(fksource.PollSource); !ok {
t.Fatalf("BuildInput(nws_forecast_narrative) type = %T, want PollSource", in)
}
}
func TestRegisterBuiltinsRegistersNWSForecastDiscussionDriver(t *testing.T) {
reg := fksource.NewRegistry()
RegisterBuiltins(reg)
in, err := reg.BuildInput(sourceConfigForDriver("nws_forecast_discussion"))
if err != nil {
t.Fatalf("BuildInput(nws_forecast_discussion) error = %v", err)
}
if _, ok := in.(fksource.PollSource); !ok {
t.Fatalf("BuildInput(nws_forecast_discussion) type = %T, want PollSource", in)
}
}
func TestRegisterBuiltinsDoesNotRegisterLegacyNWSForecastDriver(t *testing.T) {
reg := fksource.NewRegistry()
RegisterBuiltins(reg)
_, err := reg.BuildInput(sourceConfigForDriver("nws_forecast"))
if err == nil {
t.Fatalf("BuildInput(nws_forecast) expected unknown driver error")
}
if !strings.Contains(err.Error(), `unknown source driver: "nws_forecast"`) {
t.Fatalf("error = %q, want unknown source driver for nws_forecast", err)
}
}
func TestRegisterBuiltinsRegistersAllCurrentDrivers(t *testing.T) {
reg := fksource.NewRegistry()
RegisterBuiltins(reg)
drivers := []string{
"nws_observation",
"nws_alerts",
"nws_forecast_hourly",
"nws_forecast_narrative",
"nws_forecast_discussion",
"openmeteo_observation",
"openmeteo_forecast",
"openweather_observation",
}
for _, driver := range drivers {
in, err := reg.BuildInput(sourceConfigForDriver(driver))
if err != nil {
t.Fatalf("BuildInput(%s) error = %v", driver, err)
}
if _, ok := in.(fksource.PollSource); !ok {
t.Fatalf("BuildInput(%s) type = %T, want PollSource", driver, in)
}
}
}
func sourceConfigForDriver(driver string) config.SourceConfig {
url := "https://example.invalid"
if driver == "openweather_observation" {
url = "https://example.invalid?units=metric"
}
return config.SourceConfig{
Name: "test-source",
Driver: driver,
Mode: config.SourceModePoll,
Params: map[string]any{
"url": url,
"user_agent": "test-agent",
},
}
}

View File

@@ -1,104 +0,0 @@
// FILE: ./internal/sources/common/config.go
package common
import (
"fmt"
"strings"
"gitea.maximumdirect.net/ejr/feedkit/config"
)
// This file centralizes small, boring config-validation patterns shared across
// weatherfeeder source drivers.
//
// Goal: keep driver constructors (New*Source) easy to read and consistent, while
// keeping driver-specific options in cfg.Params (feedkit remains domain-agnostic).
// HTTPSourceConfig is the standard "HTTP-polling source" config shape used across drivers.
type HTTPSourceConfig struct {
Name string
URL string
UserAgent string
}
// RequireHTTPSourceConfig enforces weatherfeeder's standard HTTP source config:
//
// - cfg.Name must be present
// - cfg.Params must be present
// - params.url must be present (accepts "url" or "URL")
// - params.user_agent must be present (accepts "user_agent" or "userAgent")
//
// We intentionally require a User-Agent for *all* sources, even when upstreams
// do not strictly require one. This keeps config uniform across providers.
func RequireHTTPSourceConfig(driver string, cfg config.SourceConfig) (HTTPSourceConfig, error) {
if strings.TrimSpace(cfg.Name) == "" {
return HTTPSourceConfig{}, fmt.Errorf("%s: name is required", driver)
}
if cfg.Params == nil {
return HTTPSourceConfig{}, fmt.Errorf("%s %q: params are required (need params.url and params.user_agent)", driver, cfg.Name)
}
url, ok := cfg.ParamString("url", "URL")
if !ok {
return HTTPSourceConfig{}, fmt.Errorf("%s %q: params.url is required", driver, cfg.Name)
}
ua, ok := cfg.ParamString("user_agent", "userAgent")
if !ok {
return HTTPSourceConfig{}, fmt.Errorf("%s %q: params.user_agent is required", driver, cfg.Name)
}
return HTTPSourceConfig{
Name: cfg.Name,
URL: url,
UserAgent: ua,
}, nil
}
// --- The helpers below remain useful for future drivers; they are not required
// --- by the observation sources after adopting RequireHTTPSourceConfig.
// RequireName ensures cfg.Name is present and non-whitespace.
func RequireName(driver string, cfg config.SourceConfig) error {
if strings.TrimSpace(cfg.Name) == "" {
return fmt.Errorf("%s: name is required", driver)
}
return nil
}
// RequireParams ensures cfg.Params is non-nil. The "want" string should be a short
// description of required keys, e.g. "need params.url and params.user_agent".
func RequireParams(driver string, cfg config.SourceConfig, want string) error {
if cfg.Params == nil {
return fmt.Errorf("%s %q: params are required (%s)", driver, cfg.Name, want)
}
return nil
}
// RequireURL returns the configured URL for a source.
// Canonical key is "url"; we also accept "URL" as a convenience.
func RequireURL(driver string, cfg config.SourceConfig) (string, error) {
if cfg.Params == nil {
return "", fmt.Errorf("%s %q: params are required (need params.url)", driver, cfg.Name)
}
u, ok := cfg.ParamString("url", "URL")
if !ok {
return "", fmt.Errorf("%s %q: params.url is required", driver, cfg.Name)
}
return u, nil
}
// RequireUserAgent returns the configured User-Agent for a source.
// Canonical key is "user_agent"; we also accept "userAgent" as a convenience.
func RequireUserAgent(driver string, cfg config.SourceConfig) (string, error) {
if cfg.Params == nil {
return "", fmt.Errorf("%s %q: params are required (need params.user_agent)", driver, cfg.Name)
}
ua, ok := cfg.ParamString("user_agent", "userAgent")
if !ok {
return "", fmt.Errorf("%s %q: params.user_agent is required", driver, cfg.Name)
}
return ua, nil
}

View File

@@ -1,37 +0,0 @@
// FILE: ./internal/sources/common/event.go
package common
import (
"time"
"gitea.maximumdirect.net/ejr/feedkit/event"
)
// SingleRawEvent constructs, validates, and returns a slice containing exactly one event.
//
// This removes the repetitive "event envelope ceremony" from individual sources.
// Sources remain responsible for:
// - fetching bytes (raw payload)
// - choosing Schema (raw schema identifier)
// - computing a stable Event.ID and (optional) EffectiveAt
func SingleRawEvent(kind event.Kind, sourceName string, schema string, id string, effectiveAt *time.Time, payload any) ([]event.Event, error) {
e := event.Event{
ID: id,
Kind: kind,
Source: sourceName,
EmittedAt: time.Now().UTC(),
EffectiveAt: effectiveAt,
// RAW schema (normalizer matches on this).
Schema: schema,
// Raw payload (usually json.RawMessage). Normalizer will decode and map to canonical model.
Payload: payload,
}
if err := e.Validate(); err != nil {
return nil, err
}
return []event.Event{e}, nil
}

View File

@@ -1,70 +0,0 @@
// FILE: ./internal/sources/common/http.go
package common
import (
"context"
"fmt"
"io"
"net/http"
"time"
)
// maxResponseBodyBytes is a hard safety limit on HTTP response bodies.
// API responses should be small, so this protects us from accidental
// or malicious large responses.
const maxResponseBodyBytes = 2 << 21 // 4 MiB
// DefaultHTTPTimeout is the standard timeout used by weatherfeeder HTTP sources.
// Individual drivers may override this if they have a specific need.
const DefaultHTTPTimeout = 10 * time.Second
// NewHTTPClient returns a simple http.Client configured with a timeout.
// If timeout <= 0, DefaultHTTPTimeout is used.
func NewHTTPClient(timeout time.Duration) *http.Client {
if timeout <= 0 {
timeout = DefaultHTTPTimeout
}
return &http.Client{Timeout: timeout}
}
func FetchBody(ctx context.Context, client *http.Client, url, userAgent, accept string) ([]byte, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
if userAgent != "" {
req.Header.Set("User-Agent", userAgent)
}
if accept != "" {
req.Header.Set("Accept", accept)
}
res, err := client.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode < 200 || res.StatusCode >= 300 {
return nil, fmt.Errorf("HTTP %s", res.Status)
}
// Read at most maxResponseBodyBytes + 1 so we can detect overflow.
limited := io.LimitReader(res.Body, maxResponseBodyBytes+1)
b, err := io.ReadAll(limited)
if err != nil {
return nil, err
}
if len(b) == 0 {
return nil, fmt.Errorf("empty response body")
}
if len(b) > maxResponseBodyBytes {
return nil, fmt.Errorf("response body too large (>%d bytes)", maxResponseBodyBytes)
}
return b, nil
}

View File

@@ -1,54 +1,153 @@
// FILE: internal/sources/nws/alerts.go
package nws package nws
import ( import (
"context" "context"
"fmt" "encoding/json"
"strings" "strings"
"time"
"gitea.maximumdirect.net/ejr/feedkit/config" "gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event" "gitea.maximumdirect.net/ejr/feedkit/event"
fksources "gitea.maximumdirect.net/ejr/feedkit/sources"
nwscommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/nws"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
) )
// AlertsSource polls an NWS alerts endpoint and emits a RAW alerts Event.
//
// It intentionally emits the *entire* upstream payload as json.RawMessage and only decodes
// minimal metadata for Event.EffectiveAt and Event.ID.
//
// Output schema:
// - standards.SchemaRawNWSAlertsV1
type AlertsSource struct { type AlertsSource struct {
name string http *fksources.HTTPSource
url string
userAgent string
} }
func NewAlertsSource(cfg config.SourceConfig) (*AlertsSource, error) { func NewAlertsSource(cfg config.SourceConfig) (*AlertsSource, error) {
if strings.TrimSpace(cfg.Name) == "" { const driver = "nws_alerts"
return nil, fmt.Errorf("nws_alerts: name is required")
} // NWS alerts responses are GeoJSON-ish; allow fallback to plain JSON as well.
if cfg.Params == nil { hs, err := fksources.NewHTTPSource(driver, cfg, "application/geo+json, application/json")
return nil, fmt.Errorf("nws_alerts %q: params are required (need params.url and params.user_agent)", cfg.Name) if err != nil {
return nil, err
} }
// Driver-specific options live in cfg.Params to keep feedkit domain-agnostic. return &AlertsSource{http: hs}, nil
// Use the typed accessor so callers cant accidentally pass non-strings to TrimSpace.
url, ok := cfg.ParamString("url", "URL")
if !ok {
return nil, fmt.Errorf("nws_alerts %q: params.url is required", cfg.Name)
}
ua, ok := cfg.ParamString("user_agent", "userAgent")
if !ok {
return nil, fmt.Errorf("nws_alerts %q: params.user_agent is required", cfg.Name)
}
return &AlertsSource{
name: cfg.Name,
url: url,
userAgent: ua,
}, nil
} }
func (s *AlertsSource) Name() string { return s.name } func (s *AlertsSource) Name() string { return s.http.Name }
// Kind is used for routing/policy. // Kinds is used for routing/policy.
// The envelope type is event.Event; payload will eventually be something like model.WeatherAlert. func (s *AlertsSource) Kinds() []event.Kind { return []event.Kind{event.Kind("alert")} }
func (s *AlertsSource) Kind() event.Kind { return event.Kind("alert") }
func (s *AlertsSource) Poll(ctx context.Context) ([]event.Event, error) { func (s *AlertsSource) Poll(ctx context.Context) ([]event.Event, error) {
_ = ctx raw, meta, changed, err := s.fetchRaw(ctx)
return nil, fmt.Errorf("nws.AlertsSource.Poll: TODO implement (url=%s)", s.url) if err != nil {
return nil, err
}
if !changed {
return nil, nil
}
// EffectiveAt policy for alerts:
// Prefer the collection-level "updated" timestamp (best dedupe signal).
// If missing, fall back to the most recent per-alert timestamp we can parse.
var effectiveAt *time.Time
switch {
case !meta.ParsedUpdated.IsZero():
t := meta.ParsedUpdated.UTC()
effectiveAt = &t
case !meta.ParsedLatestFeatureTime.IsZero():
t := meta.ParsedLatestFeatureTime.UTC()
effectiveAt = &t
}
emittedAt := time.Now().UTC()
// NWS alerts collections do not provide a stable per-snapshot ID.
// Use Source:EffectiveAt (or Source:EmittedAt fallback) for dedupe friendliness.
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("alert"),
s.http.Name,
standards.SchemaRawNWSAlertsV1,
eventID,
emittedAt,
effectiveAt,
raw,
)
}
// ---- RAW fetch + minimal metadata decode ----
// alertsMeta is a minimal view of the NWS /alerts FeatureCollection.
// We only decode fields used to set EffectiveAt deterministically.
type alertsMeta struct {
Updated string `json:"updated"`
Features []struct {
Properties struct {
Sent string `json:"sent"`
Effective string `json:"effective"`
Expires string `json:"expires"`
Ends string `json:"ends"`
} `json:"properties"`
} `json:"features"`
ParsedUpdated time.Time `json:"-"`
ParsedLatestFeatureTime time.Time `json:"-"`
}
func (s *AlertsSource) fetchRaw(ctx context.Context) (json.RawMessage, alertsMeta, bool, error) {
raw, changed, err := s.http.FetchJSONIfChanged(ctx)
if err != nil {
return nil, alertsMeta{}, false, err
}
if !changed {
return nil, alertsMeta{}, false, nil
}
var meta alertsMeta
if err := json.Unmarshal(raw, &meta); err != nil {
// If metadata decode fails, still return raw; Poll will fall back to Source:EmittedAt.
return raw, alertsMeta{}, true, nil
}
// Top-level updated (preferred).
if us := strings.TrimSpace(meta.Updated); us != "" {
if t, err := nwscommon.ParseTime(us); err == nil {
meta.ParsedUpdated = t.UTC()
}
}
// If Updated is missing/unparseable, compute a best-effort "latest" feature timestamp.
// We prefer Sent/Effective, and fall back to Expires/Ends if needed.
var latest time.Time
for _, f := range meta.Features {
candidates := []string{
f.Properties.Sent,
f.Properties.Effective,
f.Properties.Expires,
f.Properties.Ends,
}
for _, s := range candidates {
ts := strings.TrimSpace(s)
if ts == "" {
continue
}
t, err := nwscommon.ParseTime(ts)
if err != nil {
continue
}
t = t.UTC()
if latest.IsZero() || t.After(latest) {
latest = t
}
}
}
meta.ParsedLatestFeatureTime = latest
return raw, meta, true, nil
} }

View File

@@ -1,51 +0,0 @@
package nws
import (
"context"
"fmt"
"strings"
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event"
)
type ForecastSource struct {
name string
url string
userAgent string
}
func NewForecastSource(cfg config.SourceConfig) (*ForecastSource, error) {
if strings.TrimSpace(cfg.Name) == "" {
return nil, fmt.Errorf("nws_forecast: name is required")
}
if cfg.Params == nil {
return nil, fmt.Errorf("nws_forecast %q: params are required (need params.url and params.user_agent)", cfg.Name)
}
url, ok := cfg.ParamString("url", "URL")
if !ok {
return nil, fmt.Errorf("nws_forecast %q: params.url is required", cfg.Name)
}
ua, ok := cfg.ParamString("user_agent", "userAgent")
if !ok {
return nil, fmt.Errorf("nws_forecast %q: params.user_agent is required", cfg.Name)
}
return &ForecastSource{
name: cfg.Name,
url: url,
userAgent: ua,
}, nil
}
func (s *ForecastSource) Name() string { return s.name }
// Kind is used for routing/policy.
func (s *ForecastSource) Kind() event.Kind { return event.Kind("forecast") }
func (s *ForecastSource) Poll(ctx context.Context) ([]event.Event, error) {
_ = ctx
return nil, fmt.Errorf("nws.ForecastSource.Poll: TODO implement (url=%s)", s.url)
}

View File

@@ -0,0 +1,114 @@
package nws
import (
"context"
"encoding/json"
"strings"
"time"
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event"
fksources "gitea.maximumdirect.net/ejr/feedkit/sources"
nwscommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/nws"
)
const nwsForecastAccept = "application/geo+json, application/json"
type forecastSource struct {
http *fksources.HTTPSource
rawSchema string
}
type forecastMeta struct {
Properties struct {
GeneratedAt string `json:"generatedAt"`
UpdateTime string `json:"updateTime"`
Updated string `json:"updated"`
} `json:"properties"`
ParsedGeneratedAt time.Time `json:"-"`
ParsedUpdateTime time.Time `json:"-"`
}
func newForecastSource(cfg config.SourceConfig, driver, rawSchema string) (*forecastSource, error) {
hs, err := fksources.NewHTTPSource(driver, cfg, nwsForecastAccept)
if err != nil {
return nil, err
}
return &forecastSource{
http: hs,
rawSchema: rawSchema,
}, nil
}
func (s *forecastSource) Name() string { return s.http.Name }
func (s *forecastSource) Kinds() []event.Kind { return []event.Kind{event.Kind("forecast")} }
func (s *forecastSource) Poll(ctx context.Context) ([]event.Event, error) {
raw, meta, changed, err := s.fetchRaw(ctx)
if err != nil {
return nil, err
}
if !changed {
return nil, nil
}
var effectiveAt *time.Time
switch {
case !meta.ParsedGeneratedAt.IsZero():
t := meta.ParsedGeneratedAt.UTC()
effectiveAt = &t
case !meta.ParsedUpdateTime.IsZero():
t := meta.ParsedUpdateTime.UTC()
effectiveAt = &t
}
emittedAt := time.Now().UTC()
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("forecast"),
s.http.Name,
s.rawSchema,
eventID,
emittedAt,
effectiveAt,
raw,
)
}
func (s *forecastSource) fetchRaw(ctx context.Context) (json.RawMessage, forecastMeta, bool, error) {
raw, changed, err := s.http.FetchJSONIfChanged(ctx)
if err != nil {
return nil, forecastMeta{}, false, err
}
if !changed {
return nil, forecastMeta{}, false, nil
}
var meta forecastMeta
if err := json.Unmarshal(raw, &meta); err != nil {
return raw, forecastMeta{}, true, nil
}
genStr := strings.TrimSpace(meta.Properties.GeneratedAt)
if genStr != "" {
if t, err := nwscommon.ParseTime(genStr); err == nil {
meta.ParsedGeneratedAt = t.UTC()
}
}
updStr := strings.TrimSpace(meta.Properties.UpdateTime)
if updStr == "" {
updStr = strings.TrimSpace(meta.Properties.Updated)
}
if updStr != "" {
if t, err := nwscommon.ParseTime(updStr); err == nil {
meta.ParsedUpdateTime = t.UTC()
}
}
return raw, meta, true, nil
}

View File

@@ -0,0 +1,68 @@
package nws
import (
"context"
"time"
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event"
fksources "gitea.maximumdirect.net/ejr/feedkit/sources"
nwscommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/nws"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
// ForecastDiscussionSource polls an NWS forecast discussion HTML page and emits a RAW discussion Event.
//
// Output schema:
// - standards.SchemaRawNWSForecastDiscussionV1
type ForecastDiscussionSource struct {
http *fksources.HTTPSource
}
func NewForecastDiscussionSource(cfg config.SourceConfig) (*ForecastDiscussionSource, error) {
const driver = "nws_forecast_discussion"
hs, err := fksources.NewHTTPSource(driver, cfg, "text/html, application/xhtml+xml")
if err != nil {
return nil, err
}
return &ForecastDiscussionSource{http: hs}, nil
}
func (s *ForecastDiscussionSource) Name() string { return s.http.Name }
func (s *ForecastDiscussionSource) Kinds() []event.Kind {
return []event.Kind{event.Kind("forecast_discussion")}
}
func (s *ForecastDiscussionSource) Poll(ctx context.Context) ([]event.Event, error) {
body, changed, err := s.http.FetchBytesIfChanged(ctx)
if err != nil {
return nil, err
}
if !changed {
return nil, nil
}
rawHTML := string(body)
parsed, err := nwscommon.ParseForecastDiscussionHTML(rawHTML)
if err != nil {
return nil, err
}
issuedAt := parsed.IssuedAt.UTC()
effectiveAt := &issuedAt
emittedAt := time.Now().UTC()
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("forecast_discussion"),
s.http.Name,
standards.SchemaRawNWSForecastDiscussionV1,
eventID,
emittedAt,
effectiveAt,
rawHTML,
)
}

View File

@@ -0,0 +1,138 @@
package nws
import (
"context"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
func TestForecastDiscussionSourcePollEmitsExpectedEvent(t *testing.T) {
rawHTML := loadForecastDiscussionSampleHTML(t)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html; charset=utf-8")
_, _ = w.Write([]byte(rawHTML))
}))
defer srv.Close()
src, err := NewForecastDiscussionSource(forecastDiscussionSourceConfig(srv.URL))
if err != nil {
t.Fatalf("NewForecastDiscussionSource() error = %v", err)
}
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("forecast_discussion") {
t.Fatalf("Kinds() = %#v, want [forecast_discussion]", got)
}
events, err := src.Poll(context.Background())
if err != nil {
t.Fatalf("Poll() error = %v", err)
}
if len(events) != 1 {
t.Fatalf("Poll() len = %d, want 1", len(events))
}
got := events[0]
if got.Kind != event.Kind("forecast_discussion") {
t.Fatalf("Kind = %q, want forecast_discussion", got.Kind)
}
if got.Schema != standards.SchemaRawNWSForecastDiscussionV1 {
t.Fatalf("Schema = %q, want %q", got.Schema, standards.SchemaRawNWSForecastDiscussionV1)
}
wantEffectiveAt := time.Date(2026, 3, 28, 19, 24, 0, 0, time.UTC)
if got.EffectiveAt == nil || !got.EffectiveAt.Equal(wantEffectiveAt) {
t.Fatalf("EffectiveAt = %v, want %s", got.EffectiveAt, wantEffectiveAt.Format(time.RFC3339))
}
payload, ok := got.Payload.(string)
if !ok {
t.Fatalf("Payload type = %T, want string", got.Payload)
}
if payload != rawHTML {
t.Fatalf("Payload did not preserve exact HTML")
}
}
func TestForecastDiscussionSourcePollReturnsNoEventsWhenUnchanged(t *testing.T) {
rawHTML := loadForecastDiscussionSampleHTML(t)
const etag = `"discussion-v1"`
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("If-None-Match") == etag {
w.WriteHeader(http.StatusNotModified)
return
}
w.Header().Set("ETag", etag)
_, _ = w.Write([]byte(rawHTML))
}))
defer srv.Close()
src, err := NewForecastDiscussionSource(forecastDiscussionSourceConfig(srv.URL))
if err != nil {
t.Fatalf("NewForecastDiscussionSource() error = %v", err)
}
first, err := src.Poll(context.Background())
if err != nil {
t.Fatalf("first Poll() error = %v", err)
}
if len(first) != 1 {
t.Fatalf("first Poll() len = %d, want 1", len(first))
}
second, err := src.Poll(context.Background())
if err != nil {
t.Fatalf("second Poll() error = %v", err)
}
if len(second) != 0 {
t.Fatalf("second Poll() len = %d, want 0", len(second))
}
}
func TestForecastDiscussionSourcePollRejectsInvalidHTML(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte("<html><body><div>missing discussion block</div></body></html>"))
}))
defer srv.Close()
src, err := NewForecastDiscussionSource(forecastDiscussionSourceConfig(srv.URL))
if err != nil {
t.Fatalf("NewForecastDiscussionSource() error = %v", err)
}
_, err = src.Poll(context.Background())
if err == nil {
t.Fatalf("Poll() error = nil, want error")
}
if !strings.Contains(err.Error(), "glossaryProduct") {
t.Fatalf("error = %q, want glossaryProduct context", err)
}
}
func forecastDiscussionSourceConfig(url string) config.SourceConfig {
return config.SourceConfig{
Name: "test-forecast-discussion-source",
Driver: "nws_forecast_discussion",
Mode: config.SourceModePoll,
Params: map[string]any{
"url": url,
"user_agent": "test-agent",
},
}
}
func loadForecastDiscussionSampleHTML(t *testing.T) string {
t.Helper()
path := filepath.Join("..", "..", "providers", "nws", "testdata", "forecast_discussion_sample.html")
b, err := os.ReadFile(path)
if err != nil {
t.Fatalf("os.ReadFile(%q) error = %v", path, err)
}
return string(b)
}

View File

@@ -0,0 +1,28 @@
// FILE: internal/sources/nws/forecast_hourly.go
package nws
import (
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
// HourlyForecastSource polls an NWS hourly forecast endpoint and emits a RAW forecast Event.
//
// It intentionally emits the *entire* upstream payload as json.RawMessage and only decodes
// minimal metadata for Event.EffectiveAt and Event.ID.
//
// Output schema (current implementation):
// - standards.SchemaRawNWSHourlyForecastV1
type HourlyForecastSource struct {
*forecastSource
}
func NewHourlyForecastSource(cfg config.SourceConfig) (*HourlyForecastSource, error) {
const driver = "nws_forecast_hourly"
src, err := newForecastSource(cfg, driver, standards.SchemaRawNWSHourlyForecastV1)
if err != nil {
return nil, err
}
return &HourlyForecastSource{forecastSource: src}, nil
}

View File

@@ -0,0 +1,28 @@
// FILE: internal/sources/nws/forecast_narrative.go
package nws
import (
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
// NarrativeForecastSource polls an NWS narrative forecast endpoint and emits a RAW forecast Event.
//
// It intentionally emits the *entire* upstream payload as json.RawMessage and only decodes
// minimal metadata for Event.EffectiveAt and Event.ID.
//
// Output schema:
// - standards.SchemaRawNWSNarrativeForecastV1
type NarrativeForecastSource struct {
*forecastSource
}
func NewNarrativeForecastSource(cfg config.SourceConfig) (*NarrativeForecastSource, error) {
const driver = "nws_forecast_narrative"
src, err := newForecastSource(cfg, driver, standards.SchemaRawNWSNarrativeForecastV1)
if err != nil {
return nil, err
}
return &NarrativeForecastSource{forecastSource: src}, nil
}

View File

@@ -0,0 +1,189 @@
package nws
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
type forecastPoller interface {
Poll(ctx context.Context) ([]event.Event, error)
}
func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
tests := []struct {
name string
driver string
wantSchema string
newSource func(config.SourceConfig) (forecastPoller, error)
}{
{
name: "hourly",
driver: "nws_forecast_hourly",
wantSchema: standards.SchemaRawNWSHourlyForecastV1,
newSource: func(cfg config.SourceConfig) (forecastPoller, error) {
return NewHourlyForecastSource(cfg)
},
},
{
name: "narrative",
driver: "nws_forecast_narrative",
wantSchema: standards.SchemaRawNWSNarrativeForecastV1,
newSource: func(cfg config.SourceConfig) (forecastPoller, error) {
return NewNarrativeForecastSource(cfg)
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`{"properties":{"generatedAt":"2026-03-28T12:00:00Z","updateTime":"2026-03-28T11:00:00Z"}}`))
}))
defer srv.Close()
src, err := tt.newSource(forecastSourceConfig(tt.driver, srv.URL))
if err != nil {
t.Fatalf("newSource() error = %v", err)
}
if ks, ok := src.(interface{ Kinds() []event.Kind }); !ok {
t.Fatalf("source does not implement Kinds()")
} else if gotKinds := ks.Kinds(); len(gotKinds) != 1 || gotKinds[0] != event.Kind("forecast") {
t.Fatalf("Kinds() = %#v, want [forecast]", gotKinds)
}
got, err := src.Poll(context.Background())
if err != nil {
t.Fatalf("Poll() error = %v", err)
}
if len(got) != 1 {
t.Fatalf("Poll() len = %d, want 1", len(got))
}
if got[0].Schema != tt.wantSchema {
t.Fatalf("Poll() schema = %q, want %q", got[0].Schema, tt.wantSchema)
}
if got[0].Kind != event.Kind("forecast") {
t.Fatalf("Poll() kind = %q, want forecast", got[0].Kind)
}
wantEffectiveAt := time.Date(2026, 3, 28, 12, 0, 0, 0, time.UTC)
if got[0].EffectiveAt == nil || !got[0].EffectiveAt.Equal(wantEffectiveAt) {
t.Fatalf("Poll() effectiveAt = %v, want %s", got[0].EffectiveAt, wantEffectiveAt)
}
})
}
}
func TestForecastSourcePollEffectiveAtFallbackOrder(t *testing.T) {
tests := []struct {
name string
body string
wantEffectiveAt *time.Time
}{
{
name: "updateTime fallback",
body: `{"properties":{"updateTime":"2026-03-28T11:00:00Z"}}`,
wantEffectiveAt: func() *time.Time {
t := time.Date(2026, 3, 28, 11, 0, 0, 0, time.UTC)
return &t
}(),
},
{
name: "updated fallback",
body: `{"properties":{"updated":"2026-03-28T10:00:00Z"}}`,
wantEffectiveAt: func() *time.Time {
t := time.Date(2026, 3, 28, 10, 0, 0, 0, time.UTC)
return &t
}(),
},
{
name: "omitted when metadata lacks timestamps",
body: `{"properties":{}}`,
wantEffectiveAt: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(tt.body))
}))
defer srv.Close()
src, err := NewHourlyForecastSource(forecastSourceConfig("nws_forecast_hourly", srv.URL))
if err != nil {
t.Fatalf("NewHourlyForecastSource() error = %v", err)
}
got, err := src.Poll(context.Background())
if err != nil {
t.Fatalf("Poll() error = %v", err)
}
if len(got) != 1 {
t.Fatalf("Poll() len = %d, want 1", len(got))
}
if tt.wantEffectiveAt == nil {
if got[0].EffectiveAt != nil {
t.Fatalf("Poll() effectiveAt = %v, want nil", got[0].EffectiveAt)
}
return
}
if got[0].EffectiveAt == nil || !got[0].EffectiveAt.Equal(*tt.wantEffectiveAt) {
t.Fatalf("Poll() effectiveAt = %v, want %s", got[0].EffectiveAt, *tt.wantEffectiveAt)
}
})
}
}
func TestForecastSourcePollMetadataDecodeFailureStillEmitsRawEvent(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`not-json`))
}))
defer srv.Close()
src, err := NewNarrativeForecastSource(forecastSourceConfig("nws_forecast_narrative", srv.URL))
if err != nil {
t.Fatalf("NewNarrativeForecastSource() error = %v", err)
}
got, err := src.Poll(context.Background())
if err != nil {
t.Fatalf("Poll() error = %v", err)
}
if len(got) != 1 {
t.Fatalf("Poll() len = %d, want 1", len(got))
}
if got[0].EffectiveAt != nil {
t.Fatalf("Poll() effectiveAt = %v, want nil", got[0].EffectiveAt)
}
if got[0].Schema != standards.SchemaRawNWSNarrativeForecastV1 {
t.Fatalf("Poll() schema = %q, want %q", got[0].Schema, standards.SchemaRawNWSNarrativeForecastV1)
}
raw, ok := got[0].Payload.(json.RawMessage)
if !ok {
t.Fatalf("Poll() payload type = %T, want json.RawMessage", got[0].Payload)
}
if string(raw) != "not-json" {
t.Fatalf("Poll() payload = %q, want %q", string(raw), "not-json")
}
}
func forecastSourceConfig(driver, url string) config.SourceConfig {
return config.SourceConfig{
Name: "test-forecast-source",
Driver: driver,
Mode: config.SourceModePoll,
Params: map[string]any{
"url": url,
"user_agent": "test-agent",
},
}
}

View File

@@ -4,72 +4,43 @@ package nws
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"fmt"
"net/http"
"strings" "strings"
"time" "time"
"gitea.maximumdirect.net/ejr/feedkit/config" "gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event" "gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/common" fksources "gitea.maximumdirect.net/ejr/feedkit/sources"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/standards" nwscommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/nws"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
) )
// ObservationSource polls an NWS station observation endpoint and emits a RAW observation Event. // ObservationSource polls an NWS station observation endpoint and emits a RAW observation Event.
//
// This corresponds to URLs like:
//
// https://api.weather.gov/stations/KSTL/observations/latest
type ObservationSource struct { type ObservationSource struct {
name string http *fksources.HTTPSource
url string
userAgent string
client *http.Client
} }
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) { func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
const driver = "nws_observation" const driver = "nws_observation"
c, err := common.RequireHTTPSourceConfig(driver, cfg) hs, err := fksources.NewHTTPSource(driver, cfg, "application/geo+json, application/json")
if err != nil { if err != nil {
return nil, err return nil, err
} }
return &ObservationSource{ return &ObservationSource{http: hs}, nil
name: c.Name,
url: c.URL,
userAgent: c.UserAgent,
client: common.NewHTTPClient(common.DefaultHTTPTimeout),
}, nil
} }
func (s *ObservationSource) Name() string { return s.name } func (s *ObservationSource) Name() string { return s.http.Name }
// Kind is used for routing/policy. func (s *ObservationSource) Kinds() []event.Kind { return []event.Kind{event.Kind("observation")} }
// We keep Kind canonical (observation) even for raw events; Schema differentiates raw vs canonical.
func (s *ObservationSource) Kind() event.Kind { return event.Kind("observation") }
// Poll fetches NWS "latest observation" and emits exactly one RAW Event.
// The RAW payload is json.RawMessage and Schema is standards.SchemaRawNWSObservationV1.
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) { func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
raw, meta, err := s.fetchRaw(ctx) raw, meta, changed, err := s.fetchRaw(ctx)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if !changed {
// Event.ID must be set BEFORE normalization (feedkit requires it). return nil, nil
// Prefer NWS-provided "id" (stable URL). Fallback to a stable computed key.
eventID := strings.TrimSpace(meta.ID)
if eventID == "" {
ts := meta.ParsedTimestamp
if ts.IsZero() {
ts = time.Now().UTC()
}
station := strings.TrimSpace(meta.StationID)
if station == "" {
station = "UNKNOWN"
}
eventID = fmt.Sprintf("nws:observation:%s:%s:%s", s.name, station, ts.UTC().Format(time.RFC3339Nano))
} }
// EffectiveAt is optional; for observations its naturally the observation timestamp. // EffectiveAt is optional; for observations its naturally the observation timestamp.
@@ -79,11 +50,15 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
effectiveAt = &t effectiveAt = &t
} }
return common.SingleRawEvent( emittedAt := time.Now().UTC()
s.Kind(), eventID := fksources.DefaultEventID(meta.ID, s.http.Name, effectiveAt, emittedAt)
s.name,
return fksources.SingleEvent(
event.Kind("observation"),
s.http.Name,
standards.SchemaRawNWSObservationV1, standards.SchemaRawNWSObservationV1,
eventID, eventID,
emittedAt,
effectiveAt, effectiveAt,
raw, raw,
) )
@@ -91,42 +66,36 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
// ---- RAW fetch + minimal metadata decode ---- // ---- RAW fetch + minimal metadata decode ----
// observationMeta is a *minimal* decode of the NWS payload used only to build
// a stable Event.ID and a useful EffectiveAt for the envelope.
type observationMeta struct { type observationMeta struct {
ID string `json:"id"` ID string `json:"id"`
Properties struct { Properties struct {
StationID string `json:"stationId"`
Timestamp string `json:"timestamp"` Timestamp string `json:"timestamp"`
} `json:"properties"` } `json:"properties"`
// Convenience fields populated after decode.
ParsedTimestamp time.Time `json:"-"` ParsedTimestamp time.Time `json:"-"`
StationID string `json:"-"`
} }
func (s *ObservationSource) fetchRaw(ctx context.Context) (json.RawMessage, observationMeta, error) { func (s *ObservationSource) fetchRaw(ctx context.Context) (json.RawMessage, observationMeta, bool, error) {
b, err := common.FetchBody(ctx, s.client, s.url, s.userAgent, "application/geo+json, application/json") raw, changed, err := s.http.FetchJSONIfChanged(ctx)
if err != nil { if err != nil {
return nil, observationMeta{}, fmt.Errorf("nws_observation %q: %w", s.name, err) return nil, observationMeta{}, false, err
}
if !changed {
return nil, observationMeta{}, false, nil
} }
raw := json.RawMessage(b)
var meta observationMeta var meta observationMeta
if err := json.Unmarshal(b, &meta); err != nil { if err := json.Unmarshal(raw, &meta); err != nil {
// If metadata decode fails, still return raw; envelope will fall back to computed ID. // If metadata decode fails, still return raw; envelope will fall back to Source:EffectiveAt.
return raw, observationMeta{}, nil return raw, observationMeta{}, true, nil
} }
meta.StationID = strings.TrimSpace(meta.Properties.StationID)
tsStr := strings.TrimSpace(meta.Properties.Timestamp) tsStr := strings.TrimSpace(meta.Properties.Timestamp)
if tsStr != "" { if tsStr != "" {
if t, err := time.Parse(time.RFC3339, tsStr); err == nil { if t, err := nwscommon.ParseTime(tsStr); err == nil {
meta.ParsedTimestamp = t meta.ParsedTimestamp = t.UTC()
} }
} }
return raw, meta, nil return raw, meta, true, nil
} }

View File

@@ -0,0 +1,66 @@
package nws
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event"
)
func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
var call int
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
call++
switch call {
case 1:
w.Header().Set("ETag", `"obs-v1"`)
_, _ = w.Write([]byte(`{"id":"obs-1","properties":{"timestamp":"2026-03-28T12:00:00Z"}}`))
case 2:
if got := r.Header.Get("If-None-Match"); got != `"obs-v1"` {
t.Fatalf("second request If-None-Match = %q", got)
}
w.WriteHeader(http.StatusNotModified)
default:
t.Fatalf("unexpected call count %d", call)
}
}))
defer srv.Close()
src, err := NewObservationSource(config.SourceConfig{
Name: "NWSObservationTest",
Driver: "nws_observation",
Mode: config.SourceModePoll,
Params: map[string]any{
"url": srv.URL,
"user_agent": "test-agent",
},
})
if err != nil {
t.Fatalf("NewObservationSource() error = %v", err)
}
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("observation") {
t.Fatalf("Kinds() = %#v, want [observation]", got)
}
first, err := src.Poll(context.Background())
if err != nil {
t.Fatalf("first Poll() error = %v", err)
}
if len(first) != 1 {
t.Fatalf("first Poll() len = %d, want 1", len(first))
}
if first[0].Kind != event.Kind("observation") {
t.Fatalf("first Poll() kind = %q", first[0].Kind)
}
second, err := src.Poll(context.Background())
if err != nil {
t.Fatalf("second Poll() error = %v", err)
}
if len(second) != 0 {
t.Fatalf("second Poll() len = %d, want 0", len(second))
}
}

View File

@@ -0,0 +1,116 @@
package openmeteo
import (
"context"
"encoding/json"
"strings"
"time"
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event"
fksources "gitea.maximumdirect.net/ejr/feedkit/sources"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/openmeteo"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
// ForecastSource polls an Open-Meteo hourly forecast endpoint and emits one RAW Forecast Event.
type ForecastSource struct {
http *fksources.HTTPSource
}
func NewForecastSource(cfg config.SourceConfig) (*ForecastSource, error) {
const driver = "openmeteo_forecast"
hs, err := fksources.NewHTTPSource(driver, cfg, "application/json")
if err != nil {
return nil, err
}
return &ForecastSource{http: hs}, nil
}
func (s *ForecastSource) Name() string { return s.http.Name }
func (s *ForecastSource) Kinds() []event.Kind { return []event.Kind{event.Kind("forecast")} }
func (s *ForecastSource) Poll(ctx context.Context) ([]event.Event, error) {
raw, meta, changed, err := s.fetchRaw(ctx)
if err != nil {
return nil, err
}
if !changed {
return nil, nil
}
// Open-Meteo does not expose a true "issued at" timestamp for forecast runs.
// We use current.time when present; otherwise we fall back to the first hourly time
// as a proxy for the start of the forecast horizon.
var effectiveAt *time.Time
if !meta.ParsedTimestamp.IsZero() {
t := meta.ParsedTimestamp.UTC()
effectiveAt = &t
}
emittedAt := time.Now().UTC()
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("forecast"),
s.http.Name,
standards.SchemaRawOpenMeteoHourlyForecastV1,
eventID,
emittedAt,
effectiveAt,
raw,
)
}
// ---- RAW fetch + minimal metadata decode ----
type forecastMeta struct {
Timezone string `json:"timezone"`
UTCOffsetSeconds int `json:"utc_offset_seconds"`
Current struct {
Time string `json:"time"`
} `json:"current"`
Hourly struct {
Time []string `json:"time"`
} `json:"hourly"`
ParsedTimestamp time.Time `json:"-"`
}
func (s *ForecastSource) fetchRaw(ctx context.Context) (json.RawMessage, forecastMeta, bool, error) {
raw, changed, err := s.http.FetchJSONIfChanged(ctx)
if err != nil {
return nil, forecastMeta{}, false, err
}
if !changed {
return nil, forecastMeta{}, false, nil
}
var meta forecastMeta
if err := json.Unmarshal(raw, &meta); err != nil {
// If metadata decode fails, still return raw; Poll will fall back to Source:EmittedAt.
return raw, forecastMeta{}, true, nil
}
ts := strings.TrimSpace(meta.Current.Time)
if ts == "" {
for _, v := range meta.Hourly.Time {
if ts = strings.TrimSpace(v); ts != "" {
break
}
}
}
if ts != "" {
if t, err := openmeteo.ParseTime(ts, meta.Timezone, meta.UTCOffsetSeconds); err == nil {
meta.ParsedTimestamp = t.UTC()
}
}
return raw, meta, true, nil
}

View File

@@ -4,59 +4,42 @@ package openmeteo
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"fmt"
"net/http"
"strings"
"time" "time"
"gitea.maximumdirect.net/ejr/feedkit/config" "gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event" "gitea.maximumdirect.net/ejr/feedkit/event"
fksources "gitea.maximumdirect.net/ejr/feedkit/sources"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/openmeteo" "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/openmeteo"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/common" "gitea.maximumdirect.net/ejr/weatherfeeder/standards"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/standards"
) )
// ObservationSource polls an Open-Meteo endpoint and emits one RAW Observation Event. // ObservationSource polls an Open-Meteo endpoint and emits one RAW Observation Event.
type ObservationSource struct { type ObservationSource struct {
name string http *fksources.HTTPSource
url string
userAgent string
client *http.Client
} }
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) { func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
const driver = "openmeteo_observation" const driver = "openmeteo_observation"
// We require params.user_agent for uniformity across sources (even though Open-Meteo hs, err := fksources.NewHTTPSource(driver, cfg, "application/json")
// itself does not strictly require a special User-Agent).
c, err := common.RequireHTTPSourceConfig(driver, cfg)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return &ObservationSource{ return &ObservationSource{http: hs}, nil
name: c.Name,
url: c.URL,
userAgent: c.UserAgent,
client: common.NewHTTPClient(common.DefaultHTTPTimeout),
}, nil
} }
func (s *ObservationSource) Name() string { return s.name } func (s *ObservationSource) Name() string { return s.http.Name }
func (s *ObservationSource) Kind() event.Kind { return event.Kind("observation") } func (s *ObservationSource) Kinds() []event.Kind { return []event.Kind{event.Kind("observation")} }
// Poll fetches Open-Meteo "current" and emits exactly one RAW Event.
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) { func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
raw, meta, err := s.fetchRaw(ctx) raw, meta, changed, err := s.fetchRaw(ctx)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if !changed {
eventID := buildEventID(s.name, meta) return nil, nil
if strings.TrimSpace(eventID) == "" {
// Extremely defensive fallback: keep the envelope valid no matter what.
eventID = fmt.Sprintf("openmeteo:current:%s:%s", s.name, time.Now().UTC().Format(time.RFC3339Nano))
} }
var effectiveAt *time.Time var effectiveAt *time.Time
@@ -65,11 +48,15 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
effectiveAt = &t effectiveAt = &t
} }
return common.SingleRawEvent( emittedAt := time.Now().UTC()
s.Kind(), eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
s.name,
return fksources.SingleEvent(
event.Kind("observation"),
s.http.Name,
standards.SchemaRawOpenMeteoCurrentV1, standards.SchemaRawOpenMeteoCurrentV1,
eventID, eventID,
emittedAt,
effectiveAt, effectiveAt,
raw, raw,
) )
@@ -78,10 +65,8 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
// ---- RAW fetch + minimal metadata decode ---- // ---- RAW fetch + minimal metadata decode ----
type openMeteoMeta struct { type openMeteoMeta struct {
Latitude float64 `json:"latitude"` Timezone string `json:"timezone"`
Longitude float64 `json:"longitude"` UTCOffsetSeconds int `json:"utc_offset_seconds"`
Timezone string `json:"timezone"`
UTCOffsetSeconds int `json:"utc_offset_seconds"`
Current struct { Current struct {
Time string `json:"time"` Time string `json:"time"`
@@ -90,41 +75,24 @@ type openMeteoMeta struct {
ParsedTimestamp time.Time `json:"-"` ParsedTimestamp time.Time `json:"-"`
} }
func (s *ObservationSource) fetchRaw(ctx context.Context) (json.RawMessage, openMeteoMeta, error) { func (s *ObservationSource) fetchRaw(ctx context.Context) (json.RawMessage, openMeteoMeta, bool, error) {
b, err := common.FetchBody(ctx, s.client, s.url, s.userAgent, "application/json") raw, changed, err := s.http.FetchJSONIfChanged(ctx)
if err != nil { if err != nil {
return nil, openMeteoMeta{}, fmt.Errorf("openmeteo_observation %q: %w", s.name, err) return nil, openMeteoMeta{}, false, err
}
if !changed {
return nil, openMeteoMeta{}, false, nil
} }
raw := json.RawMessage(b)
var meta openMeteoMeta var meta openMeteoMeta
if err := json.Unmarshal(b, &meta); err != nil { if err := json.Unmarshal(raw, &meta); err != nil {
// If metadata decode fails, still return raw; envelope will fall back to computed ID without EffectiveAt. // If metadata decode fails, still return raw; envelope will omit EffectiveAt.
return raw, openMeteoMeta{}, nil return raw, openMeteoMeta{}, true, nil
} }
// Best effort: compute a stable EffectiveAt + event ID component.
// If parsing fails, we simply omit EffectiveAt and fall back to time.Now() in buildEventID.
if t, err := openmeteo.ParseTime(meta.Current.Time, meta.Timezone, meta.UTCOffsetSeconds); err == nil { if t, err := openmeteo.ParseTime(meta.Current.Time, meta.Timezone, meta.UTCOffsetSeconds); err == nil {
meta.ParsedTimestamp = t.UTC() meta.ParsedTimestamp = t.UTC()
} }
return raw, meta, nil return raw, meta, true, nil
}
func buildEventID(sourceName string, meta openMeteoMeta) string {
locKey := ""
if meta.Latitude != 0 || meta.Longitude != 0 {
locKey = fmt.Sprintf("coord:%.5f,%.5f", meta.Latitude, meta.Longitude)
} else {
locKey = "loc:unknown"
}
ts := meta.ParsedTimestamp
if ts.IsZero() {
ts = time.Now().UTC()
}
return fmt.Sprintf("openmeteo:current:%s:%s:%s", sourceName, locKey, ts.Format(time.RFC3339Nano))
} }

View File

@@ -0,0 +1,44 @@
package openmeteo
import (
"testing"
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event"
)
func TestObservationSourceAdvertisesKinds(t *testing.T) {
src, err := NewObservationSource(config.SourceConfig{
Name: "openmeteo-observation-test",
Driver: "openmeteo_observation",
Mode: config.SourceModePoll,
Params: map[string]any{
"url": "https://example.invalid",
"user_agent": "test-agent",
},
})
if err != nil {
t.Fatalf("NewObservationSource() error = %v", err)
}
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("observation") {
t.Fatalf("Kinds() = %#v, want [observation]", got)
}
}
func TestForecastSourceAdvertisesKinds(t *testing.T) {
src, err := NewForecastSource(config.SourceConfig{
Name: "openmeteo-forecast-test",
Driver: "openmeteo_forecast",
Mode: config.SourceModePoll,
Params: map[string]any{
"url": "https://example.invalid",
"user_agent": "test-agent",
},
})
if err != nil {
t.Fatalf("NewForecastSource() error = %v", err)
}
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("forecast") {
t.Fatalf("Kinds() = %#v, want [forecast]", got)
}
}

View File

@@ -5,68 +5,49 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"net/http"
"net/url"
"strings"
"time" "time"
"gitea.maximumdirect.net/ejr/feedkit/config" "gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event" "gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/common" fksources "gitea.maximumdirect.net/ejr/feedkit/sources"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/standards" owcommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/openweather"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
) )
// ObservationSource polls the OpenWeatherMap "Current weather" endpoint and emits a RAW observation Event.
//
// IMPORTANT UNIT POLICY (weatherfeeder convention):
// OpenWeather changes units based on the `units` query parameter but does NOT include the unit
// system in the response body. To keep normalization deterministic, this driver *requires*
// `units=metric`. If absent (or non-metric), the driver returns an error.
type ObservationSource struct { type ObservationSource struct {
name string http *fksources.HTTPSource
url string
userAgent string
client *http.Client
} }
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) { func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
const driver = "openweather_observation" const driver = "openweather_observation"
c, err := common.RequireHTTPSourceConfig(driver, cfg) hs, err := fksources.NewHTTPSource(driver, cfg, "application/json")
if err != nil { if err != nil {
return nil, err return nil, err
} }
if err := requireMetricUnits(c.URL); err != nil { if err := owcommon.RequireMetricUnits(hs.URL); err != nil {
return nil, fmt.Errorf("openweather_observation %q: %w", c.Name, err) return nil, fmt.Errorf("%s %q: %w", hs.Driver, hs.Name, err)
} }
return &ObservationSource{ return &ObservationSource{http: hs}, nil
name: c.Name,
url: c.URL,
userAgent: c.UserAgent,
client: common.NewHTTPClient(common.DefaultHTTPTimeout),
}, nil
} }
func (s *ObservationSource) Name() string { return s.name } func (s *ObservationSource) Name() string { return s.http.Name }
func (s *ObservationSource) Kind() event.Kind { return event.Kind("observation") } func (s *ObservationSource) Kinds() []event.Kind { return []event.Kind{event.Kind("observation")} }
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) { func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
// Re-check policy defensively (in case the URL is mutated after construction). if err := owcommon.RequireMetricUnits(s.http.URL); err != nil {
if err := requireMetricUnits(s.url); err != nil { return nil, fmt.Errorf("%s %q: %w", s.http.Driver, s.http.Name, err)
return nil, fmt.Errorf("openweather_observation %q: %w", s.name, err)
} }
raw, meta, err := s.fetchRaw(ctx) raw, meta, changed, err := s.fetchRaw(ctx)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if !changed {
eventID := buildEventID(s.name, meta) return nil, nil
if strings.TrimSpace(eventID) == "" {
eventID = fmt.Sprintf("openweather:current:%s:%s", s.name, time.Now().UTC().Format(time.RFC3339Nano))
} }
var effectiveAt *time.Time var effectiveAt *time.Time
@@ -75,11 +56,15 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
effectiveAt = &t effectiveAt = &t
} }
return common.SingleRawEvent( emittedAt := time.Now().UTC()
s.Kind(), eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
s.name,
return fksources.SingleEvent(
event.Kind("observation"),
s.http.Name,
standards.SchemaRawOpenWeatherCurrentV1, standards.SchemaRawOpenWeatherCurrentV1,
eventID, eventID,
emittedAt,
effectiveAt, effectiveAt,
raw, raw,
) )
@@ -90,68 +75,26 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
type openWeatherMeta struct { type openWeatherMeta struct {
Dt int64 `json:"dt"` // unix seconds, UTC Dt int64 `json:"dt"` // unix seconds, UTC
ID int64 `json:"id"`
Name string `json:"name"`
Coord struct {
Lon float64 `json:"lon"`
Lat float64 `json:"lat"`
} `json:"coord"`
ParsedTimestamp time.Time `json:"-"` ParsedTimestamp time.Time `json:"-"`
} }
func (s *ObservationSource) fetchRaw(ctx context.Context) (json.RawMessage, openWeatherMeta, error) { func (s *ObservationSource) fetchRaw(ctx context.Context) (json.RawMessage, openWeatherMeta, bool, error) {
b, err := common.FetchBody(ctx, s.client, s.url, s.userAgent, "application/json") raw, changed, err := s.http.FetchJSONIfChanged(ctx)
if err != nil { if err != nil {
return nil, openWeatherMeta{}, fmt.Errorf("openweather_observation %q: %w", s.name, err) return nil, openWeatherMeta{}, false, err
}
if !changed {
return nil, openWeatherMeta{}, false, nil
} }
raw := json.RawMessage(b)
var meta openWeatherMeta var meta openWeatherMeta
if err := json.Unmarshal(b, &meta); err != nil { if err := json.Unmarshal(raw, &meta); err != nil {
return raw, openWeatherMeta{}, nil return raw, openWeatherMeta{}, true, nil
} }
if meta.Dt > 0 { if meta.Dt > 0 {
meta.ParsedTimestamp = time.Unix(meta.Dt, 0).UTC() meta.ParsedTimestamp = time.Unix(meta.Dt, 0).UTC()
} }
return raw, meta, nil return raw, meta, true, nil
}
func buildEventID(sourceName string, meta openWeatherMeta) string {
locKey := ""
if meta.ID != 0 {
locKey = fmt.Sprintf("city:%d", meta.ID)
} else if meta.Coord.Lat != 0 || meta.Coord.Lon != 0 {
locKey = fmt.Sprintf("coord:%.5f,%.5f", meta.Coord.Lat, meta.Coord.Lon)
} else {
locKey = "loc:unknown"
}
ts := meta.ParsedTimestamp
if ts.IsZero() {
ts = time.Now().UTC()
}
return fmt.Sprintf("openweather:current:%s:%s:%s", sourceName, locKey, ts.Format(time.RFC3339Nano))
}
func requireMetricUnits(rawURL string) error {
u, err := url.Parse(strings.TrimSpace(rawURL))
if err != nil {
return fmt.Errorf("invalid url %q: %w", rawURL, err)
}
units := strings.ToLower(strings.TrimSpace(u.Query().Get("units")))
if units != "metric" {
if units == "" {
units = "(missing; defaults to standard)"
}
return fmt.Errorf("url must include units=metric (got units=%s)", units)
}
return nil
} }

View File

@@ -0,0 +1,26 @@
package openweather
import (
"testing"
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event"
)
func TestObservationSourceAdvertisesKinds(t *testing.T) {
src, err := NewObservationSource(config.SourceConfig{
Name: "openweather-observation-test",
Driver: "openweather_observation",
Mode: config.SourceModePoll,
Params: map[string]any{
"url": "https://example.invalid?units=metric",
"user_agent": "test-agent",
},
})
if err != nil {
t.Fatalf("NewObservationSource() error = %v", err)
}
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("observation") {
t.Fatalf("Kinds() = %#v, want [observation]", got)
}
}

View File

@@ -1,127 +0,0 @@
package standards
import "gitea.maximumdirect.net/ejr/weatherfeeder/internal/model"
// This file provides small, shared helper functions for reasoning about WMO codes.
// These are intentionally "coarse" categories that are useful for business logic,
// dashboards, and alerting decisions.
//
// Example uses:
// - jogging suitability: precipitation? thunderstorm? freezing precip?
// - quick glance: "is it cloudy?" "is there any precip?"
// - downstream normalizers / aggregators
func IsThunderstorm(code model.WMOCode) bool {
switch code {
case 95, 96, 99:
return true
default:
return false
}
}
func IsHail(code model.WMOCode) bool {
switch code {
case 96, 99:
return true
default:
return false
}
}
func IsFog(code model.WMOCode) bool {
switch code {
case 45, 48:
return true
default:
return false
}
}
// IsPrecipitation returns true if the code represents any precipitation
// (drizzle, rain, snow, showers, etc.).
func IsPrecipitation(code model.WMOCode) bool {
switch code {
// Drizzle
case 51, 53, 55, 56, 57:
return true
// Rain
case 61, 63, 65, 66, 67:
return true
// Snow
case 71, 73, 75, 77:
return true
// Showers
case 80, 81, 82, 85, 86:
return true
// Thunderstorm (often includes rain/hail)
case 95, 96, 99:
return true
default:
return false
}
}
func IsRainFamily(code model.WMOCode) bool {
switch code {
// Drizzle + freezing drizzle
case 51, 53, 55, 56, 57:
return true
// Rain + freezing rain
case 61, 63, 65, 66, 67:
return true
// Rain showers
case 80, 81, 82:
return true
// Thunderstorm often implies rain
case 95, 96, 99:
return true
default:
return false
}
}
func IsSnowFamily(code model.WMOCode) bool {
switch code {
// Snow and related
case 71, 73, 75, 77:
return true
// Snow showers
case 85, 86:
return true
default:
return false
}
}
// IsFreezingPrecip returns true if the code represents freezing drizzle/rain.
func IsFreezingPrecip(code model.WMOCode) bool {
switch code {
case 56, 57, 66, 67:
return true
default:
return false
}
}
// IsSkyOnly returns true for codes that represent "sky condition only"
// (clear/mostly/partly/cloudy) rather than fog/precip/etc.
func IsSkyOnly(code model.WMOCode) bool {
switch code {
case 0, 1, 2, 3:
return true
default:
return false
}
}

78
model/alert.go Normal file
View File

@@ -0,0 +1,78 @@
// FILE: model/alert.go
package model
import "time"
// WeatherAlertRun is a snapshot of *active* alerts for a location as-of a point in time.
//
// This mirrors WeatherForecastRun's "one issued snapshot -> many contained items" shape:
//
// - A single run may contain zero, one, or many alerts.
// - Runs are intended to be immutable snapshots (“provider asserted X at AsOf”).
//
// Normalizers should prefer to set AsOf from a provider-supplied “updated/generated” timestamp.
// If unavailable, AsOf may be set to the poll/emit time as a fallback.
type WeatherAlertRun struct {
// Optional location metadata (provider-dependent).
LocationID string `json:"locationId,omitempty"`
LocationName string `json:"locationName,omitempty"`
// AsOf is when the provider asserted this alert snapshot is current (required).
AsOf time.Time `json:"asOf"`
// Optional spatial context.
Latitude *float64 `json:"latitude,omitempty"`
Longitude *float64 `json:"longitude,omitempty"`
// Active alerts contained in this snapshot (order is provider-dependent).
Alerts []WeatherAlert `json:"alerts"`
}
// WeatherAlert is the canonical representation of a single alert.
//
// This is intentionally a “useful subset” of rich provider payloads.
// Normalizers may populate ProviderExtras for structured provider-specific fields
// that dont cleanly fit the canonical shape.
type WeatherAlert struct {
// Provider-stable identifier (often a URL/URI).
ID string `json:"id"`
// Classification / headline fields.
Event string `json:"event,omitempty"`
Headline string `json:"headline,omitempty"`
Severity string `json:"severity,omitempty"` // e.g. Extreme/Severe/Moderate/Minor/Unknown
Urgency string `json:"urgency,omitempty"` // e.g. Immediate/Expected/Future/Past/Unknown
Certainty string `json:"certainty,omitempty"` // e.g. Observed/Likely/Possible/Unlikely/Unknown
Status string `json:"status,omitempty"` // e.g. Actual/Exercise/Test/System/Unknown
MessageType string `json:"messageType,omitempty"` // e.g. Alert/Update/Cancel
Category string `json:"category,omitempty"` // e.g. Met/Geo/Safety/Rescue/Fire/Health/Env/Transport/Infra/CBRNE/Other
Response string `json:"response,omitempty"` // e.g. Shelter/Evacuate/Prepare/Execute/Avoid/Monitor/Assess/AllClear/None
// Narrative.
Description string `json:"description,omitempty"`
Instruction string `json:"instruction,omitempty"`
// Timing (all optional; provider-dependent).
Sent *time.Time `json:"sent,omitempty"`
Effective *time.Time `json:"effective,omitempty"`
Onset *time.Time `json:"onset,omitempty"`
Expires *time.Time `json:"expires,omitempty"`
// Scope / affected area.
AreaDescription string `json:"areaDescription,omitempty"` // often a provider string
// Provenance.
SenderName string `json:"senderName,omitempty"`
References []AlertReference `json:"references,omitempty"`
}
// AlertReference is a reference to a related alert (updates, replacements, etc.).
type AlertReference struct {
ID string `json:"id,omitempty"` // provider reference ID/URI
Identifier string `json:"identifier,omitempty"` // provider identifier string, if distinct
Sender string `json:"sender,omitempty"`
Sent *time.Time `json:"sent,omitempty"`
}

10
model/doc.go Normal file
View File

@@ -0,0 +1,10 @@
// FILE: model/doc.go
// Package model defines weatherfeeder's canonical domain payload types.
//
// These structs are emitted as the Payload of canonical events (schemas "weather.*.vN").
// JSON tags are treated as part of the wire contract for sinks (stdout today; others later).
//
// Compatibility guidance:
// - Prefer additive changes.
// - Avoid renaming/removing fields without a schema version bump.
package model

108
model/forecast.go Normal file
View File

@@ -0,0 +1,108 @@
// FILE: model/forecast.go
package model
import "time"
// ForecastProduct distinguishes *what kind* of forecast a provider is offering.
// This is intentionally canonical and not provider-nomenclature.
type ForecastProduct string
const (
// ForecastProductHourly is a sub-daily forecast where each period is typically ~1 hour.
ForecastProductHourly ForecastProduct = "hourly"
// ForecastProductNarrative is a human-oriented sequence of periods like
// "Tonight", "Friday", "Friday Night" with variable duration.
//
// (NWS "/forecast" looks like this; it is not strictly “daily”.)
ForecastProductNarrative ForecastProduct = "narrative"
// ForecastProductDaily is a calendar-day (or day-bucketed) forecast where a period
// commonly carries min/max values (Open-Meteo daily, many others).
ForecastProductDaily ForecastProduct = "daily"
)
// WeatherForecastRun is a single issued forecast snapshot for a location and product.
//
// Design goals:
// - Immutable snapshot semantics: “provider asserted X at IssuedAt”.
// - Provider-independent schema: normalize many upstreams into one shape.
// - Retrieval-friendly: periods are inside the run, but can be stored/indexed separately.
type WeatherForecastRun struct {
// Identity / metadata (aligned with WeatherObservations StationID/StationName/Timestamp).
LocationID string `json:"locationId,omitempty"`
LocationName string `json:"locationName,omitempty"`
IssuedAt time.Time `json:"issuedAt"` // required: when this run was generated/issued
// Some providers include both a generated time and a later update time.
// Keep UpdatedAt optional; many providers wont supply it.
UpdatedAt *time.Time `json:"updatedAt,omitempty"`
// What sort of forecast this run represents (hourly vs narrative vs daily).
Product ForecastProduct `json:"product"`
// Optional spatial context. Many providers are fundamentally lat/lon-based.
Latitude *float64 `json:"latitude,omitempty"`
Longitude *float64 `json:"longitude,omitempty"`
// Kept to align with WeatherObservation and because elevation is sometimes important
// for interpreting temps/precip, even if not always supplied by the provider.
ElevationMeters *float64 `json:"elevationMeters,omitempty"`
// The forecast periods contained in this issued run. Order should be chronological.
Periods []WeatherForecastPeriod `json:"periods"`
}
// WeatherForecastPeriod is a forecast for a specific valid interval [StartTime, EndTime).
//
// Conceptually, it mirrors WeatherObservation (condition + measurements), but:
// / - It has a time *range* (start/end) instead of a single timestamp.
// / - It adds forecast-specific fields like probability/amount of precip.
// / - It supports min/max for “daily” products (and other aggregated periods).
type WeatherForecastPeriod struct {
// Identity / validity window (required)
StartTime time.Time `json:"startTime"`
EndTime time.Time `json:"endTime"`
// Human-facing period label (e.g., "Tonight", "Friday"). Often empty for hourly.
Name string `json:"name,omitempty"`
// Canonical day/night hint (aligned with WeatherObservation.IsDay).
// Providers vary in whether they explicitly include this.
IsDay *bool `json:"isDay,omitempty"`
// Canonical internal representation (provider-independent).
// Like WeatherObservation, this is required; use an “unknown” WMOCode if unmappable.
ConditionCode WMOCode `json:"conditionCode"`
// Human-facing narrative summary for this period.
TextDescription string `json:"textDescription,omitempty"`
// Core predicted measurements (nullable; units align with WeatherObservation)
TemperatureC *float64 `json:"temperatureC,omitempty"`
// For aggregated products (notably “daily”), providers may supply min/max.
TemperatureCMin *float64 `json:"temperatureCMin,omitempty"`
TemperatureCMax *float64 `json:"temperatureCMax,omitempty"`
DewpointC *float64 `json:"dewpointC,omitempty"`
RelativeHumidityPercent *float64 `json:"relativeHumidityPercent,omitempty"`
WindDirectionDegrees *float64 `json:"windDirectionDegrees,omitempty"`
WindSpeedKmh *float64 `json:"windSpeedKmh,omitempty"`
WindGustKmh *float64 `json:"windGustKmh,omitempty"`
BarometricPressurePa *float64 `json:"barometricPressurePa,omitempty"`
VisibilityMeters *float64 `json:"visibilityMeters,omitempty"`
ApparentTemperatureC *float64 `json:"apparentTemperatureC,omitempty"`
CloudCoverPercent *float64 `json:"cloudCoverPercent,omitempty"`
// Precipitation (forecast-specific). Keep these generic and provider-independent.
ProbabilityOfPrecipitationPercent *float64 `json:"probabilityOfPrecipitationPercent,omitempty"`
// Quantitative precip is not universally available, but OpenWeather/Open-Meteo often supply it.
// Use liquid-equivalent mm for interoperability.
PrecipitationAmountMm *float64 `json:"precipitationAmountMm,omitempty"`
SnowfallDepthMM *float64 `json:"snowfallDepthMm,omitempty"`
// Optional extras that some providers supply and downstream might care about.
UVIndex *float64 `json:"uvIndex,omitempty"`
}

View File

@@ -0,0 +1,40 @@
package model
import "time"
// ForecastDiscussionProduct distinguishes the discussion bulletin family.
//
// Today weatherfeeder only normalizes Area Forecast Discussion (AFD) products,
// but this remains a distinct type so additional discussion-like products can be
// added without changing the payload field type.
type ForecastDiscussionProduct string
const (
ForecastDiscussionProductAFD ForecastDiscussionProduct = "afd"
)
// WeatherForecastDiscussion is a canonical issued discussion bulletin for an NWS office.
//
// Unlike WeatherForecastRun, this is authored narrative text rather than a sequence
// of forecast periods.
type WeatherForecastDiscussion struct {
OfficeID string `json:"officeId,omitempty"`
OfficeName string `json:"officeName,omitempty"`
Product ForecastDiscussionProduct `json:"product"`
IssuedAt time.Time `json:"issuedAt"`
UpdatedAt *time.Time `json:"updatedAt,omitempty"`
KeyMessages []string `json:"keyMessages,omitempty"`
ShortTerm *WeatherForecastDiscussionSection `json:"shortTerm,omitempty"`
LongTerm *WeatherForecastDiscussionSection `json:"longTerm,omitempty"`
}
// WeatherForecastDiscussionSection is a fixed prose section within a discussion bulletin.
type WeatherForecastDiscussionSection struct {
Qualifier string `json:"qualifier,omitempty"`
IssuedAt *time.Time `json:"issuedAt,omitempty"`
Text string `json:"text,omitempty"`
}

36
model/observation.go Normal file
View File

@@ -0,0 +1,36 @@
// FILE: model/observation.go
package model
import "time"
type WeatherObservation struct {
// Identity / metadata
StationID string `json:"stationId,omitempty"`
StationName string `json:"stationName,omitempty"`
Timestamp time.Time `json:"timestamp"`
// Canonical internal representation (provider-independent).
ConditionCode WMOCode `json:"conditionCode"`
IsDay *bool `json:"isDay,omitempty"`
TextDescription string `json:"textDescription,omitempty"`
// Core measurements (nullable)
TemperatureC *float64 `json:"temperatureC,omitempty"`
DewpointC *float64 `json:"dewpointC,omitempty"`
WindDirectionDegrees *float64 `json:"windDirectionDegrees,omitempty"`
WindSpeedKmh *float64 `json:"windSpeedKmh,omitempty"`
WindGustKmh *float64 `json:"windGustKmh,omitempty"`
BarometricPressurePa *float64 `json:"barometricPressurePa,omitempty"`
VisibilityMeters *float64 `json:"visibilityMeters,omitempty"`
RelativeHumidityPercent *float64 `json:"relativeHumidityPercent,omitempty"`
ApparentTemperatureC *float64 `json:"apparentTemperatureC,omitempty"`
PresentWeather []PresentWeather `json:"presentWeather,omitempty"`
}
type PresentWeather struct {
Raw map[string]any `json:"raw,omitempty"`
}

View File

@@ -5,10 +5,19 @@
// - Schema identifiers and versioning conventions (see schema.go). // - Schema identifiers and versioning conventions (see schema.go).
// - Canonical interpretations / cross-provider mappings that are not specific to a // - Canonical interpretations / cross-provider mappings that are not specific to a
// single upstream API (e.g., shared code tables, text heuristics, unit policy). // single upstream API (e.g., shared code tables, text heuristics, unit policy).
// - Wire-format conventions for canonical payloads.
// //
// Standards are used by both sources and normalizers. Keep this package free of // Standards are used by both sources and normalizers. Keep this package free of
// provider-specific logic and free of dependencies on internal/sources/* or // provider-specific logic and free of dependencies on internal/sources/* or
// internal/normalizers/* to avoid import cycles. // internal/normalizers/* to avoid import cycles.
// //
// Provider-specific decoding and mapping lives in internal/normalizers/<provider>. // Wire-format conventions
// -----------------------
// For readability and stability, canonical payloads (weather.* schemas) should not emit
// noisy floating-point representations. weatherfeeder enforces this by rounding float
// values in canonical payloads to 4 digits after the decimal point at normalization
// finalization time.
//
// Provider-specific decoding helpers and quirks live in internal/providers/<provider>.
// Normalizer implementations and canonical mapping logic live in internal/normalizers/<provider>.
package standards package standards

View File

@@ -15,6 +15,17 @@ const (
SchemaRawOpenMeteoCurrentV1 = "raw.openmeteo.current.v1" SchemaRawOpenMeteoCurrentV1 = "raw.openmeteo.current.v1"
SchemaRawOpenWeatherCurrentV1 = "raw.openweather.current.v1" SchemaRawOpenWeatherCurrentV1 = "raw.openweather.current.v1"
SchemaRawNWSHourlyForecastV1 = "raw.nws.hourly.forecast.v1"
SchemaRawNWSNarrativeForecastV1 = "raw.nws.narrative.forecast.v1"
SchemaRawNWSForecastDiscussionV1 = "raw.nws.forecast_discussion.v1"
SchemaRawOpenMeteoHourlyForecastV1 = "raw.openmeteo.hourly.forecast.v1"
SchemaRawOpenWeatherHourlyForecastV1 = "raw.openweather.hourly.forecast.v1"
SchemaRawNWSAlertsV1 = "raw.nws.alerts.v1"
// Canonical domain schemas (emitted after normalization). // Canonical domain schemas (emitted after normalization).
SchemaWeatherObservationV1 = "weather.observation.v1" SchemaWeatherObservationV1 = "weather.observation.v1"
SchemaWeatherForecastV1 = "weather.forecast.v1"
SchemaWeatherForecastDiscussionV1 = "weather.forecast_discussion.v1"
SchemaWeatherAlertV1 = "weather.alert.v1"
) )

View File

@@ -1,9 +1,18 @@
package standards package standards
// This file provides small, shared helper functions for reasoning about WMO codes.
// These are intentionally "coarse" categories that are useful for business logic,
// dashboards, and alerting decisions.
//
// Example uses:
// - jogging suitability: precipitation? thunderstorm? freezing precip?
// - quick glance: "is it cloudy?" "is there any precip?"
// - downstream normalizers / aggregators
import ( import (
"fmt" "fmt"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/model" "gitea.maximumdirect.net/ejr/weatherfeeder/model"
) )
type WMODescription struct { type WMODescription struct {
@@ -12,7 +21,7 @@ type WMODescription struct {
} }
// WMODescriptions is the canonical internal mapping of WMO code -> day/night text. // WMODescriptions is the canonical internal mapping of WMO code -> day/night text.
// These are used to populate model.WeatherObservation.ConditionText. // These are used to populate canonical text fields derived from WMO codes.
var WMODescriptions = map[model.WMOCode]WMODescription{ var WMODescriptions = map[model.WMOCode]WMODescription{
0: {Day: "Sunny", Night: "Clear"}, 0: {Day: "Sunny", Night: "Clear"},
1: {Day: "Mainly Sunny", Night: "Mainly Clear"}, 1: {Day: "Mainly Sunny", Night: "Mainly Clear"},
@@ -47,7 +56,8 @@ var WMODescriptions = map[model.WMOCode]WMODescription{
// WMOText returns the canonical text description for a WMO code. // WMOText returns the canonical text description for a WMO code.
// If isDay is nil, it prefers the Day description (if present). // If isDay is nil, it prefers the Day description (if present).
// //
// This is intended to be used by drivers after they set ConditionCode. // This is intended to be used by drivers after they set ConditionCode when they
// need a human-readable description.
func WMOText(code model.WMOCode, isDay *bool) string { func WMOText(code model.WMOCode, isDay *bool) string {
if code == model.WMOUnknown { if code == model.WMOUnknown {
return "Unknown" return "Unknown"
@@ -100,3 +110,118 @@ func IsKnownWMO(code model.WMOCode) bool {
_, ok := WMODescriptions[code] _, ok := WMODescriptions[code]
return ok return ok
} }
func IsThunderstorm(code model.WMOCode) bool {
switch code {
case 95, 96, 99:
return true
default:
return false
}
}
func IsHail(code model.WMOCode) bool {
switch code {
case 96, 99:
return true
default:
return false
}
}
func IsFog(code model.WMOCode) bool {
switch code {
case 45, 48:
return true
default:
return false
}
}
// IsPrecipitation returns true if the code represents any precipitation
// (drizzle, rain, snow, showers, etc.).
func IsPrecipitation(code model.WMOCode) bool {
switch code {
// Drizzle
case 51, 53, 55, 56, 57:
return true
// Rain
case 61, 63, 65, 66, 67:
return true
// Snow
case 71, 73, 75, 77:
return true
// Showers
case 80, 81, 82, 85, 86:
return true
// Thunderstorm (often includes rain/hail)
case 95, 96, 99:
return true
default:
return false
}
}
func IsRainFamily(code model.WMOCode) bool {
switch code {
// Drizzle + freezing drizzle
case 51, 53, 55, 56, 57:
return true
// Rain + freezing rain
case 61, 63, 65, 66, 67:
return true
// Rain showers
case 80, 81, 82:
return true
// Thunderstorm often implies rain
case 95, 96, 99:
return true
default:
return false
}
}
func IsSnowFamily(code model.WMOCode) bool {
switch code {
// Snow and related
case 71, 73, 75, 77:
return true
// Snow showers
case 85, 86:
return true
default:
return false
}
}
// IsFreezingPrecip returns true if the code represents freezing drizzle/rain.
func IsFreezingPrecip(code model.WMOCode) bool {
switch code {
case 56, 57, 66, 67:
return true
default:
return false
}
}
// IsSkyOnly returns true for codes that represent "sky condition only"
// (clear/mostly/partly/cloudy) rather than fog/precip/etc.
func IsSkyOnly(code model.WMOCode) bool {
switch code {
case 0, 1, 2, 3:
return true
default:
return false
}
}