diff --git a/API.md b/API.md index c2b204e..c8c5d98 100644 --- a/API.md +++ b/API.md @@ -80,34 +80,18 @@ A `WeatherObservation` represents a point-in-time observation for a station/loca | `stationName` | string | no | Human station name | | `timestamp` | timestamp string | yes | Observation timestamp | | `conditionCode` | int | yes | WMO code (`-1` unknown) | -| `conditionText` | string | no | Canonical short condition text | | `isDay` | bool | no | Day/night hint | -| `providerRawDescription` | string | no | Provider-specific evidence text | -| `textDescription` | string | no | Legacy/transitional text description | -| `iconUrl` | string | no | Legacy/transitional icon URL | +| `textDescription` | string | no | Human-facing short description | | `temperatureC` | number | no | Celsius | | `dewpointC` | number | no | Celsius | | `windDirectionDegrees` | number | no | Degrees | | `windSpeedKmh` | number | no | km/h | | `windGustKmh` | number | no | km/h | | `barometricPressurePa` | number | no | Pascals | -| `seaLevelPressurePa` | number | no | Pascals | | `visibilityMeters` | number | no | Meters | | `relativeHumidityPercent` | number | no | Percent | | `apparentTemperatureC` | number | no | Celsius | -| `elevationMeters` | number | no | Meters | -| `rawMessage` | string | no | Provider raw message (for example METAR) | | `presentWeather` | array | no | Provider-specific structured weather fragments | -| `cloudLayers` | array | no | Cloud layer details | - -### Nested: `cloudLayers[]` - -Each `cloudLayers[]` element: - -| Field | Type | Required | Notes | -|---|---:|:---:|---| -| `baseMeters` | number | no | Cloud base altitude in meters | -| `amount` | string | no | Provider string (e.g. FEW/SCT/BKN/OVC) | ### Nested: `presentWeather[]` @@ -241,7 +225,7 @@ A run may contain zero, one, or many alerts. - Consumers **must** ignore unknown fields. - Producers (weatherfeeder) prefer **additive changes** within a schema version. -- Renames/removals/semantic breaks require a **schema version bump** (`weather.*.v2`). +- Renames/removals/semantic breaks normally require a **schema version bump** (`weather.*.v2`); pre-1.0 projects may choose in-place changes. --- @@ -259,7 +243,7 @@ A run may contain zero, one, or many alerts. "stationId": "KSTL", "timestamp": "2026-01-17T14:00:00Z", "conditionCode": 1, - "conditionText": "Mainly Sunny", + "textDescription": "Mainly Sunny", "temperatureC": 3.25, "windSpeedKmh": 18.5 } diff --git a/internal/normalizers/nws/observation.go b/internal/normalizers/nws/observation.go index 19ddd25..9324df9 100644 --- a/internal/normalizers/nws/observation.go +++ b/internal/normalizers/nws/observation.go @@ -54,14 +54,6 @@ func buildObservation(parsed nwsObservationResponse) (model.WeatherObservation, ts = t.UTC() } - cloudLayers := make([]model.CloudLayer, 0, len(parsed.Properties.CloudLayers)) - for _, cl := range parsed.Properties.CloudLayers { - cloudLayers = append(cloudLayers, model.CloudLayer{ - BaseMeters: cl.Base.Value, - Amount: cl.Amount, - }) - } - // Preserve raw presentWeather objects (for troubleshooting / drift analysis). present := make([]model.PresentWeather, 0, len(parsed.Properties.PresentWeather)) for _, pw := range parsed.Properties.PresentWeather { @@ -70,6 +62,7 @@ func buildObservation(parsed nwsObservationResponse) (model.WeatherObservation, // Decode presentWeather into typed METAR phenomena for mapping. phenomena := decodeMetarPhenomena(parsed.Properties.PresentWeather) + cloudLayers := parsed.Properties.CloudLayers providerDesc := strings.TrimSpace(parsed.Properties.TextDescription) @@ -81,9 +74,6 @@ func buildObservation(parsed nwsObservationResponse) (model.WeatherObservation, isDay = isDayFromLatLonTime(*lat, *lon, ts) } - // Canonical condition text comes from our WMO table. - canonicalText := standards.WMOText(wmo, isDay) - // Apparent temperature: prefer wind chill when both are supplied. var apparentC *float64 if parsed.Properties.WindChill.Value != nil { @@ -98,15 +88,9 @@ func buildObservation(parsed nwsObservationResponse) (model.WeatherObservation, Timestamp: ts, ConditionCode: wmo, - ConditionText: canonicalText, IsDay: isDay, - ProviderRawDescription: providerDesc, - - // Transitional / human-facing: - // keep output consistent by populating TextDescription from canonical text. - TextDescription: canonicalText, - IconURL: parsed.Properties.Icon, + TextDescription: providerDesc, TemperatureC: parsed.Properties.Temperature.Value, DewpointC: parsed.Properties.Dewpoint.Value, @@ -115,19 +99,21 @@ func buildObservation(parsed nwsObservationResponse) (model.WeatherObservation, WindSpeedKmh: parsed.Properties.WindSpeed.Value, WindGustKmh: parsed.Properties.WindGust.Value, - BarometricPressurePa: parsed.Properties.BarometricPressure.Value, - SeaLevelPressurePa: parsed.Properties.SeaLevelPressure.Value, + BarometricPressurePa: pressurePrecedenceNWS(parsed.Properties.SeaLevelPressure.Value, parsed.Properties.BarometricPressure.Value), VisibilityMeters: parsed.Properties.Visibility.Value, RelativeHumidityPercent: parsed.Properties.RelativeHumidity.Value, ApparentTemperatureC: apparentC, - ElevationMeters: parsed.Properties.Elevation.Value, - RawMessage: parsed.Properties.RawMessage, - PresentWeather: present, - CloudLayers: cloudLayers, } return obs, ts, nil } + +func pressurePrecedenceNWS(seaLevelPa, barometricPa *float64) *float64 { + if seaLevelPa != nil { + return seaLevelPa + } + return barometricPa +} diff --git a/internal/normalizers/nws/observation_test.go b/internal/normalizers/nws/observation_test.go new file mode 100644 index 0000000..c14bd73 --- /dev/null +++ b/internal/normalizers/nws/observation_test.go @@ -0,0 +1,51 @@ +package nws + +import "testing" + +func TestBuildObservationTextDescriptionAndPressurePrecedence(t *testing.T) { + barometric := 100200.0 + seaLevel := 101400.0 + + parsed := nwsObservationResponse{} + parsed.Properties.Timestamp = "2026-03-16T19:00:00Z" + parsed.Properties.TextDescription = " Overcast " + parsed.Properties.BarometricPressure.Value = &barometric + parsed.Properties.SeaLevelPressure.Value = &seaLevel + + obs, _, err := buildObservation(parsed) + if err != nil { + t.Fatalf("buildObservation() error = %v", err) + } + + if got, want := obs.TextDescription, "Overcast"; got != want { + t.Fatalf("TextDescription = %q, want %q", got, want) + } + + if obs.BarometricPressurePa == nil { + t.Fatalf("BarometricPressurePa = nil, want non-nil") + } + if got, want := *obs.BarometricPressurePa, seaLevel; got != want { + t.Fatalf("BarometricPressurePa = %v, want %v", got, want) + } +} + +func TestBuildObservationPressureFallbackToBarometric(t *testing.T) { + barometric := 99900.0 + + parsed := nwsObservationResponse{} + parsed.Properties.Timestamp = "2026-03-16T19:00:00Z" + parsed.Properties.TextDescription = "Cloudy" + parsed.Properties.BarometricPressure.Value = &barometric + + obs, _, err := buildObservation(parsed) + if err != nil { + t.Fatalf("buildObservation() error = %v", err) + } + + if obs.BarometricPressurePa == nil { + t.Fatalf("BarometricPressurePa = nil, want non-nil") + } + if got, want := *obs.BarometricPressurePa, barometric; got != want { + t.Fatalf("BarometricPressurePa = %v, want %v", got, want) + } +} diff --git a/internal/normalizers/nws/types.go b/internal/normalizers/nws/types.go index dc5327f..8719642 100644 --- a/internal/normalizers/nws/types.go +++ b/internal/normalizers/nws/types.go @@ -86,16 +86,18 @@ type nwsObservationResponse struct { // We decode these as generic maps, then optionally interpret them in metar.go. PresentWeather []map[string]any `json:"presentWeather"` - CloudLayers []struct { - Base struct { - UnitCode string `json:"unitCode"` - Value *float64 `json:"value"` - } `json:"base"` - Amount string `json:"amount"` - } `json:"cloudLayers"` + CloudLayers []nwsCloudLayer `json:"cloudLayers"` } `json:"properties"` } +type nwsCloudLayer struct { + Base struct { + UnitCode string `json:"unitCode"` + Value *float64 `json:"value"` + } `json:"base"` + Amount string `json:"amount"` +} + // nwsForecastResponse is a minimal-but-sufficient representation of the NWS // gridpoint forecast GeoJSON payload needed for mapping into model.WeatherForecastRun. // diff --git a/internal/normalizers/nws/wmo_map.go b/internal/normalizers/nws/wmo_map.go index 701cafe..72d127f 100644 --- a/internal/normalizers/nws/wmo_map.go +++ b/internal/normalizers/nws/wmo_map.go @@ -14,7 +14,7 @@ import ( // 1. METAR phenomena (presentWeather) — most reliable for precip/hazards // 2. textDescription keywords — weaker, but reusable across providers // 3. cloud layers fallback — only for sky-only conditions -func mapNWSToWMO(providerDesc string, cloudLayers []model.CloudLayer, phenomena []metarPhenomenon) model.WMOCode { +func mapNWSToWMO(providerDesc string, cloudLayers []nwsCloudLayer, phenomena []metarPhenomenon) model.WMOCode { // 1) Prefer METAR phenomena if present. if code := wmoFromPhenomena(phenomena); code != model.WMOUnknown { return code @@ -167,7 +167,7 @@ func wmoFromPhenomena(phenomena []metarPhenomenon) model.WMOCode { return model.WMOUnknown } -func wmoFromCloudLayers(cloudLayers []model.CloudLayer) model.WMOCode { +func wmoFromCloudLayers(cloudLayers []nwsCloudLayer) model.WMOCode { // NWS cloud layer amount values commonly include: // OVC, BKN, SCT, FEW, SKC, CLR, VV (vertical visibility / obscured sky) // diff --git a/internal/normalizers/openmeteo/observation.go b/internal/normalizers/openmeteo/observation.go index 99ee403..5e0709d 100644 --- a/internal/normalizers/openmeteo/observation.go +++ b/internal/normalizers/openmeteo/observation.go @@ -86,20 +86,9 @@ func buildObservation(parsed omResponse) (model.WeatherObservation, time.Time, e StationName: "Open-Meteo", Timestamp: ts, - ConditionCode: wmo, - ConditionText: canonicalText, - IsDay: isDay, - - // Open-Meteo does not provide a separate human text description for "current" - // when using weather_code; we leave provider evidence empty. - ProviderRawDescription: "", - - // Transitional / human-facing: - // keep output consistent by populating TextDescription from canonical text. + ConditionCode: wmo, + IsDay: isDay, TextDescription: canonicalText, - - // IconURL: Open-Meteo does not provide an icon URL in this endpoint. - IconURL: "", } // Measurements (all optional; only set when present). @@ -132,20 +121,13 @@ func buildObservation(parsed omResponse) (model.WeatherObservation, time.Time, e obs.WindGustKmh = &v } - if parsed.Current.SurfacePressure != nil { + if parsed.Current.PressureMSL != nil { + v := normcommon.PressurePaFromHPa(*parsed.Current.PressureMSL) + obs.BarometricPressurePa = &v + } else if parsed.Current.SurfacePressure != nil { v := normcommon.PressurePaFromHPa(*parsed.Current.SurfacePressure) obs.BarometricPressurePa = &v } - if parsed.Current.PressureMSL != nil { - v := normcommon.PressurePaFromHPa(*parsed.Current.PressureMSL) - obs.SeaLevelPressurePa = &v - } - - if parsed.Elevation != nil { - v := *parsed.Elevation - obs.ElevationMeters = &v - } - return obs, ts, nil } diff --git a/internal/normalizers/openmeteo/observation_test.go b/internal/normalizers/openmeteo/observation_test.go new file mode 100644 index 0000000..ce97b5a --- /dev/null +++ b/internal/normalizers/openmeteo/observation_test.go @@ -0,0 +1,61 @@ +package openmeteo + +import "testing" + +func TestBuildObservationTextDescriptionAndPressurePrecedence(t *testing.T) { + weatherCode := 2 + pressureMSL := 1016.0 + surfacePressure := 1009.0 + + parsed := omResponse{ + Timezone: "UTC", + UTCOffsetSeconds: 0, + Current: omCurrent{ + Time: "2026-03-16T19:00", + WeatherCode: &weatherCode, + PressureMSL: &pressureMSL, + SurfacePressure: &surfacePressure, + }, + } + + obs, _, err := buildObservation(parsed) + if err != nil { + t.Fatalf("buildObservation() error = %v", err) + } + + if got, want := obs.TextDescription, "Partly Cloudy"; got != want { + t.Fatalf("TextDescription = %q, want %q", got, want) + } + + if obs.BarometricPressurePa == nil { + t.Fatalf("BarometricPressurePa = nil, want non-nil") + } + if got, want := *obs.BarometricPressurePa, 101600.0; got != want { + t.Fatalf("BarometricPressurePa = %v, want %v", got, want) + } +} + +func TestBuildObservationPressureFallbackToSurface(t *testing.T) { + surfacePressure := 1008.0 + + parsed := omResponse{ + Timezone: "UTC", + UTCOffsetSeconds: 0, + Current: omCurrent{ + Time: "2026-03-16T19:00", + SurfacePressure: &surfacePressure, + }, + } + + obs, _, err := buildObservation(parsed) + if err != nil { + t.Fatalf("buildObservation() error = %v", err) + } + + if obs.BarometricPressurePa == nil { + t.Fatalf("BarometricPressurePa = nil, want non-nil") + } + if got, want := *obs.BarometricPressurePa, 100800.0; got != want { + t.Fatalf("BarometricPressurePa = %v, want %v", got, want) + } +} diff --git a/internal/normalizers/openweather/common.go b/internal/normalizers/openweather/common.go index 70270bc..d785f0f 100644 --- a/internal/normalizers/openweather/common.go +++ b/internal/normalizers/openweather/common.go @@ -53,15 +53,6 @@ func inferIsDay(icon string, dt, sunrise, sunset int64) *bool { return nil } -// openWeatherIconURL builds the standard OpenWeather icon URL for the given icon code. -func openWeatherIconURL(icon string) string { - icon = strings.TrimSpace(icon) - if icon == "" { - return "" - } - return fmt.Sprintf("https://openweathermap.org/img/wn/%s@2x.png", icon) -} - // openWeatherStationID returns a stable station identifier for the given response. // Prefer the OpenWeather city ID when present; otherwise, fall back to coordinates. func openWeatherStationID(parsed owmResponse) string { diff --git a/internal/normalizers/openweather/observation.go b/internal/normalizers/openweather/observation.go index 68ecb40..98ce40c 100644 --- a/internal/normalizers/openweather/observation.go +++ b/internal/normalizers/openweather/observation.go @@ -75,10 +75,10 @@ func buildObservation(parsed owmResponse) (model.WeatherObservation, time.Time, } surfacePa := normcommon.PressurePaFromHPa(parsed.Main.Pressure) - var seaLevelPa *float64 + barometricPa := &surfacePa if parsed.Main.SeaLevel != nil { v := normcommon.PressurePaFromHPa(*parsed.Main.SeaLevel) - seaLevelPa = &v + barometricPa = &v } wsKmh := normcommon.SpeedKmhFromMps(parsed.Wind.Speed) @@ -96,9 +96,6 @@ func buildObservation(parsed owmResponse) (model.WeatherObservation, time.Time, // Condition mapping: OpenWeather condition IDs -> canonical WMO code vocabulary. wmo := mapOpenWeatherToWMO(owmID) - canonicalText := standards.WMOText(wmo, isDay) - - iconURL := openWeatherIconURL(icon) stationID := openWeatherStationID(parsed) stationName := strings.TrimSpace(parsed.Name) @@ -111,15 +108,9 @@ func buildObservation(parsed owmResponse) (model.WeatherObservation, time.Time, StationName: stationName, Timestamp: ts, - ConditionCode: wmo, - ConditionText: canonicalText, - IsDay: isDay, - - ProviderRawDescription: rawDesc, - - // Human-facing legacy fields: populate with canonical text for consistency. - TextDescription: canonicalText, - IconURL: iconURL, + ConditionCode: wmo, + IsDay: isDay, + TextDescription: rawDesc, TemperatureC: &tempC, ApparentTemperatureC: apparentC, @@ -128,8 +119,7 @@ func buildObservation(parsed owmResponse) (model.WeatherObservation, time.Time, WindSpeedKmh: &wsKmh, WindGustKmh: wgKmh, - BarometricPressurePa: &surfacePa, - SeaLevelPressurePa: seaLevelPa, + BarometricPressurePa: barometricPa, VisibilityMeters: visM, RelativeHumidityPercent: &rh, diff --git a/internal/normalizers/openweather/observation_test.go b/internal/normalizers/openweather/observation_test.go new file mode 100644 index 0000000..93e5f7d --- /dev/null +++ b/internal/normalizers/openweather/observation_test.go @@ -0,0 +1,58 @@ +package openweather + +import "testing" + +func TestBuildObservationTextDescriptionAndPressurePrecedence(t *testing.T) { + seaLevel := 1018.0 + + parsed := owmResponse{} + parsed.Dt = 1710000000 + parsed.Main.Temp = 20.0 + parsed.Main.Humidity = 45.0 + parsed.Main.Pressure = 1000.0 + parsed.Main.SeaLevel = &seaLevel + parsed.Wind.Speed = 3.0 + parsed.Weather = []owmWeather{ + {ID: 801, Main: "Clouds", Description: "few clouds", Icon: "02d"}, + } + + obs, _, err := buildObservation(parsed) + if err != nil { + t.Fatalf("buildObservation() error = %v", err) + } + + if obs.TextDescription != "few clouds" { + t.Fatalf("TextDescription = %q, want %q", obs.TextDescription, "few clouds") + } + + if obs.BarometricPressurePa == nil { + t.Fatalf("BarometricPressurePa = nil, want non-nil") + } + if got, want := *obs.BarometricPressurePa, 101800.0; got != want { + t.Fatalf("BarometricPressurePa = %v, want %v", got, want) + } +} + +func TestBuildObservationPressureFallbackToSurface(t *testing.T) { + parsed := owmResponse{} + parsed.Dt = 1710000000 + parsed.Main.Temp = 20.0 + parsed.Main.Humidity = 45.0 + parsed.Main.Pressure = 1001.0 + parsed.Wind.Speed = 3.0 + parsed.Weather = []owmWeather{ + {ID: 800, Description: "clear sky", Icon: "01d"}, + } + + obs, _, err := buildObservation(parsed) + if err != nil { + t.Fatalf("buildObservation() error = %v", err) + } + + if obs.BarometricPressurePa == nil { + t.Fatalf("BarometricPressurePa = nil, want non-nil") + } + if got, want := *obs.BarometricPressurePa, 100100.0; got != want { + t.Fatalf("BarometricPressurePa = %v, want %v", got, want) + } +} diff --git a/internal/sinks/postgres/doc.go b/internal/sinks/postgres/doc.go index a1603a5..324bbc2 100644 --- a/internal/sinks/postgres/doc.go +++ b/internal/sinks/postgres/doc.go @@ -11,7 +11,6 @@ // - weather.alert.v1 -> model.WeatherAlertRun // // Parent/child relationships: -// - observations.event_id -> observation_cloud_layers.event_id // - observations.event_id -> observation_present_weather.event_id // - forecasts.event_id -> forecast_periods.run_event_id // - alert_runs.event_id -> alerts.run_event_id @@ -22,7 +21,6 @@ // - Child primary keys use positional indexes to preserve payload order. // - Prune columns: // - observations.observed_at -// - observation_cloud_layers.observed_at // - observation_present_weather.observed_at // - forecasts.issued_at // - forecast_periods.issued_at @@ -54,33 +52,19 @@ // - station_name TEXT NULL -> payload.stationName // - observed_at TIMESTAMPTZ -> payload.timestamp // - condition_code INTEGER -> payload.conditionCode -// - condition_text TEXT NULL -> payload.conditionText // - is_day BOOLEAN NULL -> payload.isDay -// - provider_raw_description TEXT NULL -> payload.providerRawDescription // - text_description TEXT NULL -> payload.textDescription -// - icon_url TEXT NULL -> payload.iconUrl // - temperature_c DOUBLE PRECISION NULL -> payload.temperatureC // - dewpoint_c DOUBLE PRECISION NULL -> payload.dewpointC // - wind_direction_degrees DOUBLE PRECISION NULL -> payload.windDirectionDegrees // - wind_speed_kmh DOUBLE PRECISION NULL -> payload.windSpeedKmh // - wind_gust_kmh DOUBLE PRECISION NULL -> payload.windGustKmh // - barometric_pressure_pa DOUBLE PRECISION NULL -> payload.barometricPressurePa -// - sea_level_pressure_pa DOUBLE PRECISION NULL -> payload.seaLevelPressurePa // - visibility_meters DOUBLE PRECISION NULL -> payload.visibilityMeters // - relative_humidity_percent DOUBLE PRECISION NULL -> payload.relativeHumidityPercent // - apparent_temperature_c DOUBLE PRECISION NULL -> payload.apparentTemperatureC -// - elevation_meters DOUBLE PRECISION NULL -> payload.elevationMeters -// - raw_message TEXT NULL -> payload.rawMessage // -// 2. observation_cloud_layers (PK: event_id, layer_index) -// -// - event_id TEXT -> observations.event_id / payload.cloudLayers[i] -// - layer_index INTEGER -> i (array position in payload.cloudLayers) -// - observed_at TIMESTAMPTZ -> payload.timestamp -// - base_meters DOUBLE PRECISION NULL -> payload.cloudLayers[i].baseMeters -// - amount TEXT NULL -> payload.cloudLayers[i].amount -// -// 3. observation_present_weather (PK: event_id, weather_index) +// 2. observation_present_weather (PK: event_id, weather_index) // // - event_id TEXT -> observations.event_id / payload.presentWeather[i] // - weather_index INTEGER -> i (array position in payload.presentWeather) @@ -90,7 +74,7 @@ // Note: raw_text stores compact JSON text. Consumers that need the original // object should parse raw_text as JSON. // -// 4. forecasts (PK: event_id) +// 3. forecasts (PK: event_id) // // - event_id TEXT -> event.id // - event_kind TEXT -> event.kind @@ -108,7 +92,7 @@ // - elevation_meters DOUBLE PRECISION NULL -> payload.elevationMeters // - period_count INTEGER -> len(payload.periods) // -// 5. forecast_periods (PK: run_event_id, period_index) +// 4. forecast_periods (PK: run_event_id, period_index) // // - run_event_id TEXT -> forecasts.event_id / payload.periods[i] // - period_index INTEGER -> i (array position in payload.periods) @@ -140,7 +124,7 @@ // - snowfall_depth_mm DOUBLE PRECISION NULL -> payload.periods[i].snowfallDepthMm // - uv_index DOUBLE PRECISION NULL -> payload.periods[i].uvIndex // -// 6. alert_runs (PK: event_id) +// 5. alert_runs (PK: event_id) // // - event_id TEXT -> event.id // - event_kind TEXT -> event.kind @@ -155,7 +139,7 @@ // - longitude DOUBLE PRECISION NULL -> payload.longitude // - alert_count INTEGER -> len(payload.alerts) // -// 7. alerts (PK: run_event_id, alert_index) +// 6. alerts (PK: run_event_id, alert_index) // // - run_event_id TEXT -> alert_runs.event_id / payload.alerts[i] // - alert_index INTEGER -> i (array position in payload.alerts) @@ -180,7 +164,7 @@ // - sender_name TEXT NULL -> payload.alerts[i].senderName // - reference_count INTEGER -> len(payload.alerts[i].references) // -// 8. alert_references (PK: run_event_id, alert_index, reference_index) +// 7. alert_references (PK: run_event_id, alert_index, reference_index) // // - run_event_id TEXT -> alert_runs.event_id / payload.alerts[i].references[j] // - alert_index INTEGER -> i (array position in payload.alerts) @@ -195,7 +179,7 @@ // // - WeatherObservation: // read one row from observations, then join child rows by event_id ordered by -// layer_index / weather_index to rebuild cloudLayers and presentWeather arrays. +// weather_index to rebuild presentWeather arrays. // // - WeatherForecastRun: // read one row from forecasts, then join forecast_periods by run_event_id diff --git a/internal/sinks/postgres/map.go b/internal/sinks/postgres/map.go index 4365fb5..49311cb 100644 --- a/internal/sinks/postgres/map.go +++ b/internal/sinks/postgres/map.go @@ -37,7 +37,7 @@ func mapObservationEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) { } observedAt := obs.Timestamp.UTC() - writes := make([]fksinks.PostgresWrite, 0, 1+len(obs.CloudLayers)+len(obs.PresentWeather)) + writes := make([]fksinks.PostgresWrite, 0, 1+len(obs.PresentWeather)) writes = append(writes, fksinks.PostgresWrite{ Table: tableObservations, @@ -52,39 +52,20 @@ func mapObservationEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) { "station_name": nullableString(obs.StationName), "observed_at": observedAt, "condition_code": int(obs.ConditionCode), - "condition_text": nullableString(obs.ConditionText), "is_day": nullableBool(obs.IsDay), - "provider_raw_description": nullableString(obs.ProviderRawDescription), "text_description": nullableString(obs.TextDescription), - "icon_url": nullableString(obs.IconURL), "temperature_c": nullableFloat64(obs.TemperatureC), "dewpoint_c": nullableFloat64(obs.DewpointC), "wind_direction_degrees": nullableFloat64(obs.WindDirectionDegrees), "wind_speed_kmh": nullableFloat64(obs.WindSpeedKmh), "wind_gust_kmh": nullableFloat64(obs.WindGustKmh), "barometric_pressure_pa": nullableFloat64(obs.BarometricPressurePa), - "sea_level_pressure_pa": nullableFloat64(obs.SeaLevelPressurePa), "visibility_meters": nullableFloat64(obs.VisibilityMeters), "relative_humidity_percent": nullableFloat64(obs.RelativeHumidityPercent), "apparent_temperature_c": nullableFloat64(obs.ApparentTemperatureC), - "elevation_meters": nullableFloat64(obs.ElevationMeters), - "raw_message": nullableString(obs.RawMessage), }, }) - for i, cl := range obs.CloudLayers { - writes = append(writes, fksinks.PostgresWrite{ - Table: tableObservationCloudLayers, - Values: map[string]any{ - "event_id": e.ID, - "layer_index": i, - "observed_at": observedAt, - "base_meters": nullableFloat64(cl.BaseMeters), - "amount": nullableString(cl.Amount), - }, - }) - } - for i, pw := range obs.PresentWeather { rawText, err := compactJSONText(pw.Raw) if err != nil { diff --git a/internal/sinks/postgres/map_test.go b/internal/sinks/postgres/map_test.go index dad0fd9..cdacf7e 100644 --- a/internal/sinks/postgres/map_test.go +++ b/internal/sinks/postgres/map_test.go @@ -16,28 +16,23 @@ import ( func TestMapPostgresEventObservationStructPayload(t *testing.T) { isDay := true temp := 21.5 - base := 1200.0 obs := model.WeatherObservation{ - StationID: "KSTL", - StationName: "St. Louis", - Timestamp: time.Date(2026, 3, 16, 19, 0, 0, 0, time.UTC), - ConditionCode: model.WMOCode(1), - ConditionText: "Mainly Sunny", - IsDay: &isDay, - ProviderRawDescription: "few clouds", - TextDescription: "Mainly Sunny", - IconURL: "https://example/icon.png", - TemperatureC: &temp, - CloudLayers: []model.CloudLayer{{BaseMeters: &base, Amount: "FEW"}}, - PresentWeather: []model.PresentWeather{{Raw: map[string]any{"a": 1, "b": "x"}}}, + StationID: "KSTL", + StationName: "St. Louis", + Timestamp: time.Date(2026, 3, 16, 19, 0, 0, 0, time.UTC), + ConditionCode: model.WMOCode(1), + IsDay: &isDay, + TextDescription: "few clouds", + TemperatureC: &temp, + PresentWeather: []model.PresentWeather{{Raw: map[string]any{"a": 1, "b": "x"}}}, } writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherObservationV1, "observation", obs)) if err != nil { t.Fatalf("mapPostgresEvent() error = %v", err) } - if len(writes) != 3 { - t.Fatalf("mapPostgresEvent() writes len = %d, want 3", len(writes)) + if len(writes) != 2 { + t.Fatalf("mapPostgresEvent() writes len = %d, want 2", len(writes)) } if writes[0].Table != tableObservations { t.Fatalf("writes[0].Table = %q, want %q", writes[0].Table, tableObservations) @@ -45,13 +40,10 @@ func TestMapPostgresEventObservationStructPayload(t *testing.T) { if got := writes[0].Values["station_id"]; got != "KSTL" { t.Fatalf("observations station_id = %#v, want KSTL", got) } - if writes[1].Table != tableObservationCloudLayers { - t.Fatalf("writes[1].Table = %q, want %q", writes[1].Table, tableObservationCloudLayers) + if writes[1].Table != tableObservationPresentWeather { + t.Fatalf("writes[1].Table = %q, want %q", writes[1].Table, tableObservationPresentWeather) } - if writes[2].Table != tableObservationPresentWeather { - t.Fatalf("writes[2].Table = %q, want %q", writes[2].Table, tableObservationPresentWeather) - } - if got := writes[2].Values["raw_text"]; got != `{"a":1,"b":"x"}` { + if got := writes[1].Values["raw_text"]; got != `{"a":1,"b":"x"}` { t.Fatalf("present_weather raw_text = %#v, want compact JSON", got) } diff --git a/internal/sinks/postgres/schema.go b/internal/sinks/postgres/schema.go index 3c18ce7..2927213 100644 --- a/internal/sinks/postgres/schema.go +++ b/internal/sinks/postgres/schema.go @@ -10,7 +10,6 @@ import ( const ( tableObservations = "observations" - tableObservationCloudLayers = "observation_cloud_layers" tableObservationPresentWeather = "observation_present_weather" tableForecasts = "forecasts" tableForecastPeriods = "forecast_periods" @@ -59,23 +58,17 @@ func weatherPostgresSchema() fksinks.PostgresSchema { {Name: "station_name", Type: "TEXT", Nullable: true}, {Name: "observed_at", Type: "TIMESTAMPTZ", Nullable: false}, {Name: "condition_code", Type: "INTEGER", Nullable: false}, - {Name: "condition_text", Type: "TEXT", Nullable: true}, {Name: "is_day", Type: "BOOLEAN", Nullable: true}, - {Name: "provider_raw_description", Type: "TEXT", Nullable: true}, {Name: "text_description", Type: "TEXT", Nullable: true}, - {Name: "icon_url", Type: "TEXT", Nullable: true}, {Name: "temperature_c", Type: "DOUBLE PRECISION", Nullable: true}, {Name: "dewpoint_c", Type: "DOUBLE PRECISION", Nullable: true}, {Name: "wind_direction_degrees", Type: "DOUBLE PRECISION", Nullable: true}, {Name: "wind_speed_kmh", Type: "DOUBLE PRECISION", Nullable: true}, {Name: "wind_gust_kmh", Type: "DOUBLE PRECISION", Nullable: true}, {Name: "barometric_pressure_pa", Type: "DOUBLE PRECISION", Nullable: true}, - {Name: "sea_level_pressure_pa", Type: "DOUBLE PRECISION", Nullable: true}, {Name: "visibility_meters", Type: "DOUBLE PRECISION", Nullable: true}, {Name: "relative_humidity_percent", Type: "DOUBLE PRECISION", Nullable: true}, {Name: "apparent_temperature_c", Type: "DOUBLE PRECISION", Nullable: true}, - {Name: "elevation_meters", Type: "DOUBLE PRECISION", Nullable: true}, - {Name: "raw_message", Type: "TEXT", Nullable: true}, }, PrimaryKey: []string{"event_id"}, PruneColumn: "observed_at", @@ -85,21 +78,6 @@ func weatherPostgresSchema() fksinks.PostgresSchema { {Name: "idx_wf_obs_condition_code", Columns: []string{"condition_code"}}, }, }, - { - Name: tableObservationCloudLayers, - Columns: []fksinks.PostgresColumn{ - {Name: "event_id", Type: "TEXT REFERENCES observations(event_id) ON DELETE CASCADE", Nullable: false}, - {Name: "layer_index", Type: "INTEGER", Nullable: false}, - {Name: "observed_at", Type: "TIMESTAMPTZ", Nullable: false}, - {Name: "base_meters", Type: "DOUBLE PRECISION", Nullable: true}, - {Name: "amount", Type: "TEXT", Nullable: true}, - }, - PrimaryKey: []string{"event_id", "layer_index"}, - PruneColumn: "observed_at", - Indexes: []fksinks.PostgresIndex{ - {Name: "idx_wf_obs_cloud_observed_at", Columns: []string{"observed_at"}}, - }, - }, { Name: tableObservationPresentWeather, Columns: []fksinks.PostgresColumn{ diff --git a/internal/sinks/postgres/schema_test.go b/internal/sinks/postgres/schema_test.go index 719875a..cfad385 100644 --- a/internal/sinks/postgres/schema_test.go +++ b/internal/sinks/postgres/schema_test.go @@ -61,7 +61,6 @@ func TestWeatherPostgresSchemaShape(t *testing.T) { wantTables := map[string]bool{ tableObservations: true, - tableObservationCloudLayers: true, tableObservationPresentWeather: true, tableForecasts: true, tableForecastPeriods: true, diff --git a/model/observation.go b/model/observation.go index f61a1de..57672ac 100644 --- a/model/observation.go +++ b/model/observation.go @@ -10,18 +10,9 @@ type WeatherObservation struct { Timestamp time.Time `json:"timestamp"` // Canonical internal representation (provider-independent). - ConditionCode WMOCode `json:"conditionCode"` - ConditionText string `json:"conditionText,omitempty"` - IsDay *bool `json:"isDay,omitempty"` - - // Provider-specific “evidence” for troubleshooting mapping and drift. - ProviderRawDescription string `json:"providerRawDescription,omitempty"` - - // Human-facing (legacy / transitional) - TextDescription string `json:"textDescription,omitempty"` - - // Provider-specific (legacy / transitional) - IconURL string `json:"iconUrl,omitempty"` + ConditionCode WMOCode `json:"conditionCode"` + IsDay *bool `json:"isDay,omitempty"` + TextDescription string `json:"textDescription,omitempty"` // Core measurements (nullable) TemperatureC *float64 `json:"temperatureC,omitempty"` @@ -32,22 +23,12 @@ type WeatherObservation struct { WindGustKmh *float64 `json:"windGustKmh,omitempty"` BarometricPressurePa *float64 `json:"barometricPressurePa,omitempty"` - SeaLevelPressurePa *float64 `json:"seaLevelPressurePa,omitempty"` VisibilityMeters *float64 `json:"visibilityMeters,omitempty"` RelativeHumidityPercent *float64 `json:"relativeHumidityPercent,omitempty"` ApparentTemperatureC *float64 `json:"apparentTemperatureC,omitempty"` - ElevationMeters *float64 `json:"elevationMeters,omitempty"` - RawMessage string `json:"rawMessage,omitempty"` - PresentWeather []PresentWeather `json:"presentWeather,omitempty"` - CloudLayers []CloudLayer `json:"cloudLayers,omitempty"` -} - -type CloudLayer struct { - BaseMeters *float64 `json:"baseMeters,omitempty"` - Amount string `json:"amount,omitempty"` } type PresentWeather struct { diff --git a/standards/wmo.go b/standards/wmo.go index b4f4232..caa6c4d 100644 --- a/standards/wmo.go +++ b/standards/wmo.go @@ -21,7 +21,7 @@ type WMODescription struct { } // WMODescriptions is the canonical internal mapping of WMO code -> day/night text. -// These are used to populate model.WeatherObservation.ConditionText. +// These are used to populate canonical text fields derived from WMO codes. var WMODescriptions = map[model.WMOCode]WMODescription{ 0: {Day: "Sunny", Night: "Clear"}, 1: {Day: "Mainly Sunny", Night: "Mainly Clear"}, @@ -56,7 +56,8 @@ var WMODescriptions = map[model.WMOCode]WMODescription{ // WMOText returns the canonical text description for a WMO code. // If isDay is nil, it prefers the Day description (if present). // -// This is intended to be used by drivers after they set ConditionCode. +// This is intended to be used by drivers after they set ConditionCode when they +// need a human-readable description. func WMOText(code model.WMOCode, isDay *bool) string { if code == model.WMOUnknown { return "Unknown"