ejr/weatherfeeder
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement remaining cleanup items prior to the next release
All checks were successful
ci/woodpecker/push/build-image Pipeline was successful
Details

Browse Source
This commit is contained in:
Eric Rakestraw
2026-06-11 10:17:49 -05:00
parent 8041f99782
commit 5d7f604a2c
9 changed files with 171 additions and 308 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
internal/sinks/postgres/map_test.go
View File

@@ -528,6 +528,56 @@ func TestMapPostgresEventForecastDiscussionMalformedPayload(t *testing.T) {
}
}
func TestParentEventValuesAddsEnvelopeAndPreservesProductValues(t *testing.T) {
emittedAt := time.Date(2026, 3, 16, 13, 31, 0, 0, time.FixedZone("CDT", -5*60*60))
effectiveAt := time.Date(2026, 3, 16, 13, 30, 0, 0, time.FixedZone("CDT", -5*60*60))
event := fkevent.Event{
ID: "evt-envelope",
Kind: fkevent.Kind(standards.KindForecast),
Source: "test-source",
Schema: standards.SchemaWeatherForecastV1,
EmittedAt: emittedAt,
EffectiveAt: &effectiveAt,
}
got := parentEventValues(event, map[string]any{"product_col": "product-value"})
assertParentEnvelopeValues(t, got, event)
if got["product_col"] != "product-value" {
t.Fatalf("product_col = %#v, want product-value", got["product_col"])
}
}
func TestParentEventValuesNullEffectiveAt(t *testing.T) {
base := fkevent.Event{
ID: "evt-envelope",
Kind: fkevent.Kind(standards.KindObservation),
Source: "test-source",
Schema: standards.SchemaWeatherObservationV1,
EmittedAt: time.Date(2026, 3, 16, 18, 31, 0, 0, time.UTC),
}
for _, tt := range []struct {
name string
mut func(*fkevent.Event)
}{
{name: "nil", mut: func(*fkevent.Event) {}},
{name: "zero", mut: func(event *fkevent.Event) {
zero := time.Time{}
event.EffectiveAt = &zero
}},
} {
t.Run(tt.name, func(t *testing.T) {
event := base
tt.mut(&event)
got := parentEventValues(event, nil)
if got["event_effective_at"] != nil {
t.Fatalf("event_effective_at = %#v, want nil", got["event_effective_at"])
}
})
}
}
func testEvent(schema string, kind fkevent.Kind, payload any) fkevent.Event {
effectiveAt := time.Date(2026, 3, 16, 18, 30, 0, 0, time.UTC)
return fkevent.Event{
@@ -541,6 +591,30 @@ func testEvent(schema string, kind fkevent.Kind, payload any) fkevent.Event {
}
}
func assertParentEnvelopeValues(t *testing.T, values map[string]any, event fkevent.Event) {
t.Helper()
if got := values["event_id"]; got != event.ID {
t.Fatalf("event_id = %#v, want %q", got, event.ID)
}
if got := values["event_kind"]; got != string(event.Kind) {
t.Fatalf("event_kind = %#v, want %q", got, event.Kind)
}
if got := values["event_source"]; got != event.Source {
t.Fatalf("event_source = %#v, want %q", got, event.Source)
}
if got := values["event_schema"]; got != event.Schema {
t.Fatalf("event_schema = %#v, want %q", got, event.Schema)
}
if got := values["event_emitted_at"]; got != event.EmittedAt.UTC() {
t.Fatalf("event_emitted_at = %#v, want %s", got, event.EmittedAt.UTC())
}
wantEffective := nullableTime(event.EffectiveAt)
if got := values["event_effective_at"]; got != wantEffective {
t.Fatalf("event_effective_at = %#v, want %#v", got, wantEffective)
}
}
func firstWriteForTable(writes []fksinks.PostgresWrite, table string) (fksinks.PostgresWrite, bool) {
for _, w := range writes {
if w.Table == table {

72
internal/sinks/postgres/schema.go
View File

@@ -26,13 +26,7 @@ func PostgresSchema() fksinks.PostgresSchema {
Tables: []fksinks.PostgresTable{
{
Name: tableObservations,
Columns: []fksinks.PostgresColumn{
{Name: "event_id", Type: "TEXT", Nullable: false},
{Name: "event_kind", Type: "TEXT", Nullable: false},
{Name: "event_source", Type: "TEXT", Nullable: false},
{Name: "event_schema", Type: "TEXT", Nullable: false},
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
{Name: "station_id", Type: "TEXT", Nullable: true},
{Name: "station_name", Type: "TEXT", Nullable: true},
{Name: "observed_at", Type: "TIMESTAMPTZ", Nullable: false},
@@ -48,7 +42,7 @@ func PostgresSchema() fksinks.PostgresSchema {
{Name: "visibility_meters", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "relative_humidity_percent", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "apparent_temperature_c", Type: "DOUBLE PRECISION", Nullable: true},
},
}...),
PrimaryKey: []string{"event_id"},
PruneColumn: "observed_at",
Indexes: []fksinks.PostgresIndex{
@@ -73,13 +67,7 @@ func PostgresSchema() fksinks.PostgresSchema {
},
{
Name: tableForecasts,
Columns: []fksinks.PostgresColumn{
{Name: "event_id", Type: "TEXT", Nullable: false},
{Name: "event_kind", Type: "TEXT", Nullable: false},
{Name: "event_source", Type: "TEXT", Nullable: false},
{Name: "event_schema", Type: "TEXT", Nullable: false},
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
{Name: "location_id", Type: "TEXT", Nullable: true},
{Name: "location_name", Type: "TEXT", Nullable: true},
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: false},
@@ -89,7 +77,7 @@ func PostgresSchema() fksinks.PostgresSchema {
{Name: "longitude", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "elevation_meters", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "period_count", Type: "INTEGER", Nullable: false},
},
}...),
PrimaryKey: []string{"event_id"},
PruneColumn: "issued_at",
Indexes: []fksinks.PostgresIndex{
@@ -137,13 +125,7 @@ func PostgresSchema() fksinks.PostgresSchema {
},
{
Name: tableForecastDiscussions,
Columns: []fksinks.PostgresColumn{
{Name: "event_id", Type: "TEXT", Nullable: false},
{Name: "event_kind", Type: "TEXT", Nullable: false},
{Name: "event_source", Type: "TEXT", Nullable: false},
{Name: "event_schema", Type: "TEXT", Nullable: false},
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
{Name: "office_id", Type: "TEXT", Nullable: true},
{Name: "office_name", Type: "TEXT", Nullable: true},
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: false},
@@ -156,7 +138,7 @@ func PostgresSchema() fksinks.PostgresSchema {
{Name: "long_term_issued_at", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "long_term_text", Type: "TEXT", Nullable: true},
{Name: "key_message_count", Type: "INTEGER", Nullable: false},
},
}...),
PrimaryKey: []string{"event_id"},
PruneColumn: "issued_at",
Indexes: []fksinks.PostgresIndex{
@@ -180,17 +162,11 @@ func PostgresSchema() fksinks.PostgresSchema {
},
{
Name: tableWeatherStoryRuns,
Columns: []fksinks.PostgresColumn{
{Name: "event_id", Type: "TEXT", Nullable: false},
{Name: "event_kind", Type: "TEXT", Nullable: false},
{Name: "event_source", Type: "TEXT", Nullable: false},
{Name: "event_schema", Type: "TEXT", Nullable: false},
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
{Name: "office_id", Type: "TEXT", Nullable: true},
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "story_count", Type: "INTEGER", Nullable: false},
},
}...),
PrimaryKey: []string{"event_id"},
PruneColumn: "as_of",
Indexes: []fksinks.PostgresIndex{
@@ -225,20 +201,14 @@ func PostgresSchema() fksinks.PostgresSchema {
},
{
Name: tableAlertRuns,
Columns: []fksinks.PostgresColumn{
{Name: "event_id", Type: "TEXT", Nullable: false},
{Name: "event_kind", Type: "TEXT", Nullable: false},
{Name: "event_source", Type: "TEXT", Nullable: false},
{Name: "event_schema", Type: "TEXT", Nullable: false},
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
{Name: "location_id", Type: "TEXT", Nullable: true},
{Name: "location_name", Type: "TEXT", Nullable: true},
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "latitude", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "longitude", Type: "DOUBLE PRECISION", Nullable: true},
{Name: "alert_count", Type: "INTEGER", Nullable: false},
},
}...),
PrimaryKey: []string{"event_id"},
PruneColumn: "as_of",
Indexes: []fksinks.PostgresIndex{
@@ -301,13 +271,7 @@ func PostgresSchema() fksinks.PostgresSchema {
},
{
Name: tableOutlookRuns,
Columns: []fksinks.PostgresColumn{
{Name: "event_id", Type: "TEXT", Nullable: false},
{Name: "event_kind", Type: "TEXT", Nullable: false},
{Name: "event_source", Type: "TEXT", Nullable: false},
{Name: "event_schema", Type: "TEXT", Nullable: false},
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
Columns: parentEnvelopeColumns([]fksinks.PostgresColumn{
{Name: "location_id", Type: "TEXT", Nullable: true},
{Name: "location_name", Type: "TEXT", Nullable: true},
{Name: "latitude", Type: "DOUBLE PRECISION", Nullable: true},
@@ -315,7 +279,7 @@ func PostgresSchema() fksinks.PostgresSchema {
{Name: "as_of", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "issued_at", Type: "TIMESTAMPTZ", Nullable: true},
{Name: "outlook_count", Type: "INTEGER", Nullable: false},
},
}...),
PrimaryKey: []string{"event_id"},
PruneColumn: "as_of",
Indexes: []fksinks.PostgresIndex{
@@ -362,3 +326,15 @@ func PostgresSchema() fksinks.PostgresSchema {
MapEvent: mapPostgresEvent,
}
}
func parentEnvelopeColumns(extra ...fksinks.PostgresColumn) []fksinks.PostgresColumn {
columns := []fksinks.PostgresColumn{
{Name: "event_id", Type: "TEXT", Nullable: false},
{Name: "event_kind", Type: "TEXT", Nullable: false},
{Name: "event_source", Type: "TEXT", Nullable: false},
{Name: "event_schema", Type: "TEXT", Nullable: false},
{Name: "event_emitted_at", Type: "TIMESTAMPTZ", Nullable: false},
{Name: "event_effective_at", Type: "TIMESTAMPTZ", Nullable: true},
}
return append(columns, extra...)
}

47
internal/sinks/postgres/schema_test.go
View File

@@ -1,8 +1,11 @@
package postgres
import (
"reflect"
"strings"
"testing"
fksinks "gitea.maximumdirect.net/ejr/feedkit/sinks"
)
func TestWeatherPostgresSchemaShape(t *testing.T) {
@@ -88,6 +91,28 @@ func TestWeatherPostgresSchemaIncludesWeatherStoryColumns(t *testing.T) {
}
}
func TestWeatherPostgresSchemaParentTablesStartWithEnvelopeColumns(t *testing.T) {
for _, table := range []string{
tableObservations,
tableForecasts,
tableForecastDiscussions,
tableWeatherStoryRuns,
tableAlertRuns,
tableOutlookRuns,
} {
t.Run(table, func(t *testing.T) {
columns := orderedColumnsForTable(t, table)
want := parentEnvelopeColumns()
if len(columns) < len(want) {
t.Fatalf("%s has %d columns, want at least %d", table, len(columns), len(want))
}
if !reflect.DeepEqual(columns[:len(want)], want) {
t.Fatalf("%s envelope prefix = %#v, want %#v", table, columns[:len(want)], want)
}
})
}
}
func assertTablePrimaryKey(t *testing.T, table string, want []string) {
t.Helper()
for _, tbl := range PostgresSchema().Tables {
@@ -121,20 +146,26 @@ func assertTableIndex(t *testing.T, table string, name string, want []string) {
t.Fatalf("missing table %q", table)
}
func columnsForTable(t *testing.T, table string) map[string]bool {
func orderedColumnsForTable(t *testing.T, table string) []fksinks.PostgresColumn {
t.Helper()
schema := PostgresSchema()
for _, tbl := range schema.Tables {
if tbl.Name != table {
continue
if tbl.Name == table {
return tbl.Columns
}
cols := make(map[string]bool, len(tbl.Columns))
for _, col := range tbl.Columns {
cols[col.Name] = true
}
return cols
}
t.Fatalf("missing table %q", table)
return nil
}
func columnsForTable(t *testing.T, table string) map[string]bool {
t.Helper()
ordered := orderedColumnsForTable(t, table)
cols := make(map[string]bool, len(ordered))
for _, col := range ordered {
cols[col.Name] = true
}
return cols
}