7 Commits

Author SHA1 Message Date
06d5973746 Clean up stale internal literals
Some checks failed
ci/woodpecker/push/build-image Pipeline was canceled
2026-06-11 02:23:42 +00:00
8045b27173 Move SPC provider fixture helper 2026-06-11 02:20:25 +00:00
985468c1b9 Add documentation identifier consistency tests 2026-06-11 02:18:15 +00:00
6a0b30b7c7 Centralize Postgres event envelope mapping 2026-06-11 02:15:52 +00:00
86ce4eb68c Share HTTP config parsing for multi-document sources 2026-06-11 02:13:38 +00:00
33541a71fc Table-drive source registry tests 2026-06-11 02:10:13 +00:00
b7277e0c02 Centralize weather event and driver identifiers 2026-06-11 02:08:36 +00:00
40 changed files with 528 additions and 309 deletions

View File

@@ -20,6 +20,7 @@ import (
wfnormalizers "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers"
wfsources "gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
type testInput struct {
@@ -36,10 +37,10 @@ type testKindsSource struct {
func (s testKindsSource) Kinds() []fkevent.Kind { return s.kinds }
func TestValidateSourceExpectedKindsSubsetAllowed(t *testing.T) {
sc := config.SourceConfig{Kinds: []string{"observation"}}
sc := config.SourceConfig{Kinds: []string{standards.KindObservation}}
in := testKindsSource{
testInput: testInput{name: "test"},
kinds: []fkevent.Kind{"observation", "forecast"},
kinds: []fkevent.Kind{fkevent.Kind(standards.KindObservation), fkevent.Kind(standards.KindForecast)},
}
if err := fksources.ValidateExpectedKinds(sc, in); err != nil {
@@ -48,10 +49,10 @@ func TestValidateSourceExpectedKindsSubsetAllowed(t *testing.T) {
}
func TestValidateSourceExpectedKindsMismatchFails(t *testing.T) {
sc := config.SourceConfig{Kinds: []string{"alert"}}
sc := config.SourceConfig{Kinds: []string{standards.KindAlert}}
in := testKindsSource{
testInput: testInput{name: "test"},
kinds: []fkevent.Kind{"observation", "forecast"},
kinds: []fkevent.Kind{fkevent.Kind(standards.KindObservation), fkevent.Kind(standards.KindForecast)},
}
err := fksources.ValidateExpectedKinds(sc, in)
@@ -64,7 +65,7 @@ func TestValidateSourceExpectedKindsMismatchFails(t *testing.T) {
}
func TestValidateSourceExpectedKindsNoMetadataSkipsCheck(t *testing.T) {
sc := config.SourceConfig{Kinds: []string{"alert"}}
sc := config.SourceConfig{Kinds: []string{standards.KindAlert}}
in := testInput{name: "test"}
if err := fksources.ValidateExpectedKinds(sc, in); err != nil {
@@ -111,6 +112,10 @@ func TestMaintainedConfigExamplesLoad(t *testing.T) {
func assertConfigSourcesBuildSchedulerJobs(t *testing.T, cfg *config.Config) {
t.Helper()
if len(cfg.Sources) == 0 {
t.Fatalf("config has no sources")
}
reg := fksources.NewRegistry()
wfsources.RegisterBuiltins(reg)
@@ -158,7 +163,7 @@ func TestNormalizeNoMatchPassThrough(t *testing.T) {
pl := &fkpipeline.Pipeline{Processors: chain}
in := fkevent.Event{
ID: "evt-no-match",
Kind: fkevent.Kind("observation"),
Kind: fkevent.Kind(standards.KindObservation),
Source: "test",
EmittedAt: time.Now().UTC(),
Schema: "raw.weatherfeeder.unknown.v1",
@@ -188,7 +193,7 @@ func TestDedupeDropsSecondEventWithSameID(t *testing.T) {
pl := &fkpipeline.Pipeline{Processors: chain}
in := fkevent.Event{
ID: "evt-dedupe-1",
Kind: fkevent.Kind("observation"),
Kind: fkevent.Kind(standards.KindObservation),
Source: "test",
EmittedAt: time.Now().UTC(),
Schema: "raw.weatherfeeder.unknown.v1",

View File

@@ -5,6 +5,7 @@ import (
"time"
"gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
func TestFinalizeRoundsWeatherPayloadFloats(t *testing.T) {
@@ -14,7 +15,7 @@ func TestFinalizeRoundsWeatherPayloadFloats(t *testing.T) {
in := event.Event{
ID: "evt-1",
Kind: event.Kind("observation"),
Kind: event.Kind(standards.KindObservation),
Source: "source-a",
EmittedAt: time.Date(2026, 3, 28, 12, 0, 0, 0, time.UTC),
Schema: "raw.example.v1",

View File

@@ -18,7 +18,7 @@ func TestForecastDiscussionNormalizerProducesCanonicalSchema(t *testing.T) {
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
ID: "evt-discussion-1",
Kind: event.Kind("forecast_discussion"),
Kind: event.Kind(standards.KindForecastDiscussion),
Source: "nws-discussion-test",
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
Schema: standards.SchemaRawNWSForecastDiscussionV1,
@@ -33,7 +33,7 @@ func TestForecastDiscussionNormalizerProducesCanonicalSchema(t *testing.T) {
if out.Schema != standards.SchemaWeatherForecastDiscussionV1 {
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherForecastDiscussionV1)
}
if out.Kind != event.Kind("forecast_discussion") {
if out.Kind != event.Kind(standards.KindForecastDiscussion) {
t.Fatalf("Kind = %q, want forecast_discussion", out.Kind)
}
@@ -74,7 +74,7 @@ func TestForecastDiscussionNormalizerProducesCanonicalSchema(t *testing.T) {
func TestForecastDiscussionNormalizerRejectsMissingIssueTime(t *testing.T) {
_, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
ID: "evt-discussion-bad",
Kind: event.Kind("forecast_discussion"),
Kind: event.Kind(standards.KindForecastDiscussion),
Source: "nws-discussion-test",
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
Schema: standards.SchemaRawNWSForecastDiscussionV1,
@@ -93,7 +93,7 @@ func TestForecastDiscussionNormalizerWireShapeHasNoUnexpectedKeys(t *testing.T)
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
ID: "evt-discussion-2",
Kind: event.Kind("forecast_discussion"),
Kind: event.Kind(standards.KindForecastDiscussion),
Source: "nws-discussion-test",
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
Schema: standards.SchemaRawNWSForecastDiscussionV1,

View File

@@ -182,7 +182,7 @@ func TestNormalizeForecastEventBySchemaProducesCanonicalWeatherForecastSchema(t
t.Run(tt.name, func(t *testing.T) {
out, err := normalizeForecastEventBySchema(event.Event{
ID: "evt-1",
Kind: event.Kind("forecast"),
Kind: event.Kind(standards.KindForecast),
Source: "nws-test",
EmittedAt: time.Date(2026, 3, 16, 18, 0, 0, 0, time.UTC),
Schema: tt.schema,

View File

@@ -22,7 +22,7 @@ func TestWeatherStoriesNormalizerProducesCanonicalSchemaAndMapsSample(t *testing
if out.Schema != standards.SchemaWeatherStoryV1 {
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherStoryV1)
}
if out.Kind != event.Kind("weather_story") {
if out.Kind != event.Kind(standards.KindWeatherStory) {
t.Fatalf("Kind = %q, want weather_story", out.Kind)
}
@@ -118,7 +118,7 @@ func TestWeatherStoriesNormalizerMatch(t *testing.T) {
func weatherStoriesRawEvent(payload string) event.Event {
return event.Event{
ID: "evt-weatherstories-1",
Kind: event.Kind("weather_story"),
Kind: event.Kind(standards.KindWeatherStory),
Source: "nws-weatherstories-test",
EmittedAt: time.Date(2026, 5, 30, 9, 5, 0, 0, time.UTC),
Schema: standards.SchemaRawNWSWeatherStoriesV1,

View File

@@ -22,7 +22,6 @@ const (
providerSPC = "spc"
productConvective = "convective"
outlookNormalizer = "spc convective outlook"
outlookKind = "outlook"
outlookTypeUnknown = 99
)

View File

@@ -32,7 +32,7 @@ func TestConvectiveOutlookNormalizerProducesCanonicalSchemaAndMapsSample(t *test
if out.Schema != standards.SchemaWeatherOutlookV1 {
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherOutlookV1)
}
if out.Kind != event.Kind("outlook") {
if out.Kind != event.Kind(standards.KindOutlook) {
t.Fatalf("Kind = %q, want outlook", out.Kind)
}
@@ -276,7 +276,7 @@ func spcRawEvent(t *testing.T, bundle spcprovider.RawConvectiveOutlookBundle) ev
effectiveAt := time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)
return event.Event{
ID: "evt-spc-outlook-1",
Kind: event.Kind("outlook"),
Kind: event.Kind(standards.KindOutlook),
Source: "spc-test",
EmittedAt: time.Date(2026, 6, 11, 20, 5, 0, 0, time.UTC),
EffectiveAt: &effectiveAt,

View File

@@ -0,0 +1,17 @@
package spc
import (
"os"
"path/filepath"
"testing"
)
func readTestFile(t *testing.T, name string) []byte {
t.Helper()
path := filepath.Join("testdata", name)
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read %s: %v", path, err)
}
return raw
}

View File

@@ -1,8 +1,6 @@
package spc
import (
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -78,13 +76,3 @@ func TestParseISOTimestampTrimsAndReturnsUTC(t *testing.T) {
t.Fatalf("ParseISOTimestamp() = %s, want %s", got, want)
}
}
func readTestFile(t *testing.T, name string) []byte {
t.Helper()
path := filepath.Join("testdata", name)
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read %s: %v", path, err)
}
return raw
}

View File

@@ -48,13 +48,7 @@ func mapObservationEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
writes = append(writes, fksinks.PostgresWrite{
Table: tableObservations,
Values: map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
Values: parentEventValues(e, map[string]any{
"station_id": nullableString(obs.StationID),
"station_name": nullableString(obs.StationName),
"observed_at": observedAt,
@@ -70,7 +64,7 @@ func mapObservationEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
"visibility_meters": nullableFloat64(obs.VisibilityMeters),
"relative_humidity_percent": nullableFloat64(obs.RelativeHumidityPercent),
"apparent_temperature_c": nullableFloat64(obs.ApparentTemperatureC),
},
}),
})
for i, pw := range obs.PresentWeather {
@@ -109,23 +103,17 @@ func mapForecastEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
writes = append(writes, fksinks.PostgresWrite{
Table: tableForecasts,
Values: map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
"location_id": nullableString(run.LocationID),
"location_name": nullableString(run.LocationName),
"issued_at": issuedAt,
"updated_at": nullableTime(run.UpdatedAt),
"product": string(run.Product),
"latitude": nullableFloat64(run.Latitude),
"longitude": nullableFloat64(run.Longitude),
"elevation_meters": nullableFloat64(run.ElevationMeters),
"period_count": len(run.Periods),
},
Values: parentEventValues(e, map[string]any{
"location_id": nullableString(run.LocationID),
"location_name": nullableString(run.LocationName),
"issued_at": issuedAt,
"updated_at": nullableTime(run.UpdatedAt),
"product": string(run.Product),
"latitude": nullableFloat64(run.Latitude),
"longitude": nullableFloat64(run.Longitude),
"elevation_meters": nullableFloat64(run.ElevationMeters),
"period_count": len(run.Periods),
}),
})
for i, p := range run.Periods {
@@ -186,13 +174,7 @@ func mapForecastDiscussionEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.KeyMessages))
writes = append(writes, fksinks.PostgresWrite{
Table: tableForecastDiscussions,
Values: map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
Values: parentEventValues(e, map[string]any{
"office_id": nullableString(run.OfficeID),
"office_name": nullableString(run.OfficeName),
"issued_at": issuedAt,
@@ -205,7 +187,7 @@ func mapForecastDiscussionEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error
"long_term_issued_at": longTermIssuedAt,
"long_term_text": longTermText,
"key_message_count": len(run.KeyMessages),
},
}),
})
for i, msg := range run.KeyMessages {
@@ -236,17 +218,11 @@ func mapWeatherStoryEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.Stories))
writes = append(writes, fksinks.PostgresWrite{
Table: tableWeatherStoryRuns,
Values: map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
"office_id": nullableString(run.OfficeID),
"as_of": asOf,
"story_count": len(run.Stories),
},
Values: parentEventValues(e, map[string]any{
"office_id": nullableString(run.OfficeID),
"as_of": asOf,
"story_count": len(run.Stories),
}),
})
for i, story := range run.Stories {
@@ -290,20 +266,14 @@ func mapAlertEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
writes = append(writes, fksinks.PostgresWrite{
Table: tableAlertRuns,
Values: map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
"location_id": nullableString(run.LocationID),
"location_name": nullableString(run.LocationName),
"as_of": asOf,
"latitude": nullableFloat64(run.Latitude),
"longitude": nullableFloat64(run.Longitude),
"alert_count": len(run.Alerts),
},
Values: parentEventValues(e, map[string]any{
"location_id": nullableString(run.LocationID),
"location_name": nullableString(run.LocationName),
"as_of": asOf,
"latitude": nullableFloat64(run.Latitude),
"longitude": nullableFloat64(run.Longitude),
"alert_count": len(run.Alerts),
}),
})
for i, a := range run.Alerts {
@@ -372,21 +342,15 @@ func mapOutlookEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.Outlooks))
writes = append(writes, fksinks.PostgresWrite{
Table: tableOutlookRuns,
Values: map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
"location_id": nullableString(run.LocationID),
"location_name": nullableString(run.LocationName),
"latitude": nullableFloat64(run.Latitude),
"longitude": nullableFloat64(run.Longitude),
"as_of": asOf,
"issued_at": nullableTime(run.IssuedAt),
"outlook_count": len(run.Outlooks),
},
Values: parentEventValues(e, map[string]any{
"location_id": nullableString(run.LocationID),
"location_name": nullableString(run.LocationName),
"latitude": nullableFloat64(run.Latitude),
"longitude": nullableFloat64(run.Longitude),
"as_of": asOf,
"issued_at": nullableTime(run.IssuedAt),
"outlook_count": len(run.Outlooks),
}),
})
for i, outlook := range run.Outlooks {
@@ -488,6 +452,21 @@ func decodePayload[T any](payload any) (T, error) {
return out, nil
}
func parentEventValues(e fkevent.Event, values map[string]any) map[string]any {
out := map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
}
for k, v := range values {
out[k] = v
}
return out
}
func nullableDiscussionSection(section *model.WeatherForecastDiscussionSection) (any, any, any) {
if section == nil {
return nil, nil, nil

View File

@@ -27,7 +27,7 @@ func TestMapPostgresEventObservationStructPayload(t *testing.T) {
PresentWeather: []model.PresentWeather{{Raw: map[string]any{"a": 1, "b": "x"}}},
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherObservationV1, "observation", obs))
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherObservationV1, standards.KindObservation, obs))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
@@ -74,7 +74,7 @@ func TestMapPostgresEventForecastStructPayload(t *testing.T) {
},
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, "forecast", run))
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, standards.KindForecast, run))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
@@ -122,7 +122,7 @@ func TestMapPostgresEventAlertStructPayload(t *testing.T) {
},
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherAlertV1, "alert", run))
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherAlertV1, standards.KindAlert, run))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
@@ -163,7 +163,7 @@ func TestMapPostgresEventForecastDiscussionStructPayload(t *testing.T) {
LongTerm: &model.WeatherForecastDiscussionSection{Text: "Long term text"},
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, "forecast_discussion", run))
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, standards.KindForecastDiscussion, run))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
@@ -213,7 +213,7 @@ func TestMapPostgresEventWeatherStoryStructPayload(t *testing.T) {
},
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, "weather_story", run))
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, standards.KindWeatherStory, run))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
@@ -290,7 +290,7 @@ func TestMapPostgresEventOutlookStructPayload(t *testing.T) {
},
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, "outlook", run))
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, standards.KindOutlook, run))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
@@ -332,7 +332,7 @@ func TestMapPostgresEventOutlookStructPayload(t *testing.T) {
}
func TestMapPostgresEventOutlookRejectsMissingAsOf(t *testing.T) {
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, "outlook", model.WeatherOutlookRun{}))
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, standards.KindOutlook, model.WeatherOutlookRun{}))
if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want missing asOf error")
}
@@ -381,7 +381,7 @@ func TestMapPostgresEventOutlookRejectsMissingIDAndProvider(t *testing.T) {
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
Outlooks: []model.WeatherOutlook{outlook},
}
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, "outlook", run))
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, standards.KindOutlook, run))
if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want %q", tt.wantErr)
}
@@ -405,7 +405,7 @@ func TestMapPostgresEventOutlookRejectsMissingRequiredTimes(t *testing.T) {
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-91,38],[-90,38],[-90,39],[-91,39],[-91,38]]]}`),
}},
}
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, "outlook", run))
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, standards.KindOutlook, run))
if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want missing time error")
}
@@ -430,7 +430,7 @@ func TestMapPostgresEventOutlookRejectsEmptyGeometry(t *testing.T) {
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
}},
}
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, "outlook", run))
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, standards.KindOutlook, run))
if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want geometry error")
}
@@ -440,7 +440,7 @@ func TestMapPostgresEventOutlookRejectsEmptyGeometry(t *testing.T) {
}
func TestMapPostgresEventWeatherStoryRejectsMissingAsOf(t *testing.T) {
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, "weather_story", model.WeatherStoryRun{}))
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, standards.KindWeatherStory, model.WeatherStoryRun{}))
if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want missing asOf error")
}
@@ -454,7 +454,7 @@ func TestMapPostgresEventWeatherStoryRejectsMissingStoryTimes(t *testing.T) {
AsOf: time.Date(2026, 5, 30, 9, 0, 34, 0, time.UTC),
Stories: []model.WeatherStory{{Title: "missing times"}},
}
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, "weather_story", run))
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, standards.KindWeatherStory, run))
if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want missing story times error")
}
@@ -484,7 +484,7 @@ func TestMapPostgresEventMapPayload(t *testing.T) {
t.Fatalf("json.Unmarshal() error = %v", err)
}
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, "forecast", payload))
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, standards.KindForecast, payload))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
@@ -499,7 +499,7 @@ func TestMapPostgresEventMapPayload(t *testing.T) {
}
func TestMapPostgresEventUnknownSchemaNoOp(t *testing.T) {
writes, err := mapPostgresEvent(context.Background(), testEvent("weather.unknown.v1", "observation", map[string]any{"x": 1}))
writes, err := mapPostgresEvent(context.Background(), testEvent("weather.unknown.v1", standards.KindObservation, map[string]any{"x": 1}))
if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err)
}
@@ -509,7 +509,7 @@ func TestMapPostgresEventUnknownSchemaNoOp(t *testing.T) {
}
func TestMapPostgresEventMalformedPayload(t *testing.T) {
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, "forecast", "bad"))
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, standards.KindForecast, "bad"))
if err == nil {
t.Fatalf("mapPostgresEvent() expected error for malformed payload")
}
@@ -519,7 +519,7 @@ func TestMapPostgresEventMalformedPayload(t *testing.T) {
}
func TestMapPostgresEventForecastDiscussionMalformedPayload(t *testing.T) {
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, "forecast_discussion", "bad"))
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, standards.KindForecastDiscussion, "bad"))
if err == nil {
t.Fatalf("mapPostgresEvent() expected error for malformed payload")
}

View File

@@ -16,20 +16,20 @@ type pollDriverRegistration struct {
}
var pollDriverRegistrations = []pollDriverRegistration{
{driver: "nws_observation", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewObservationSource(cfg) }},
{driver: "nws_alerts", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewAlertsSource(cfg) }},
{driver: "nws_forecast_hourly", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewHourlyForecastSource(cfg) }},
{driver: "nws_forecast_narrative", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewNarrativeForecastSource(cfg) }},
{driver: "nws_forecast_discussion", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
{driver: nws.DriverObservation, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewObservationSource(cfg) }},
{driver: nws.DriverAlerts, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewAlertsSource(cfg) }},
{driver: nws.DriverForecastHourly, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewHourlyForecastSource(cfg) }},
{driver: nws.DriverForecastNarrative, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewNarrativeForecastSource(cfg) }},
{driver: nws.DriverForecastDiscussion, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
return nws.NewForecastDiscussionSource(cfg)
}},
{driver: "nws_weatherstories", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewWeatherStoriesSource(cfg) }},
{driver: "openmeteo_observation", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewObservationSource(cfg) }},
{driver: "openmeteo_forecast", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewForecastSource(cfg) }},
{driver: "openweather_observation", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
{driver: nws.DriverWeatherStories, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewWeatherStoriesSource(cfg) }},
{driver: openmeteo.DriverObservation, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewObservationSource(cfg) }},
{driver: openmeteo.DriverForecast, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewForecastSource(cfg) }},
{driver: openweather.DriverObservation, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
return openweather.NewObservationSource(cfg)
}},
{driver: "spc_convective_outlook", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
{driver: spc.DriverConvectiveOutlook, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
return spc.NewConvectiveOutlookSource(cfg)
}},
}

View File

@@ -6,57 +6,29 @@ import (
"gitea.maximumdirect.net/ejr/feedkit/config"
fksource "gitea.maximumdirect.net/ejr/feedkit/sources"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/openweather"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/spc"
)
func TestRegisterBuiltinsRegistersNWSHourlyForecastDriver(t *testing.T) {
func TestRegisterBuiltinsRegistersCurrentPollDrivers(t *testing.T) {
reg := fksource.NewRegistry()
RegisterBuiltins(reg)
in, err := reg.BuildInput(sourceConfigForDriver("nws_forecast_hourly"))
if err != nil {
t.Fatalf("BuildInput(nws_forecast_hourly) error = %v", err)
if len(pollDriverRegistrations) == 0 {
t.Fatalf("pollDriverRegistrations is empty")
}
if _, ok := in.(fksource.PollSource); !ok {
t.Fatalf("BuildInput(nws_forecast_hourly) type = %T, want PollSource", in)
}
}
func TestRegisterBuiltinsRegistersNWSNarrativeForecastDriver(t *testing.T) {
reg := fksource.NewRegistry()
RegisterBuiltins(reg)
in, err := reg.BuildInput(sourceConfigForDriver("nws_forecast_narrative"))
if err != nil {
t.Fatalf("BuildInput(nws_forecast_narrative) error = %v", err)
}
if _, ok := in.(fksource.PollSource); !ok {
t.Fatalf("BuildInput(nws_forecast_narrative) type = %T, want PollSource", in)
}
}
func TestRegisterBuiltinsRegistersNWSForecastDiscussionDriver(t *testing.T) {
reg := fksource.NewRegistry()
RegisterBuiltins(reg)
in, err := reg.BuildInput(sourceConfigForDriver("nws_forecast_discussion"))
if err != nil {
t.Fatalf("BuildInput(nws_forecast_discussion) error = %v", err)
}
if _, ok := in.(fksource.PollSource); !ok {
t.Fatalf("BuildInput(nws_forecast_discussion) type = %T, want PollSource", in)
}
}
func TestRegisterBuiltinsRegistersNWSWeatherStoriesDriver(t *testing.T) {
reg := fksource.NewRegistry()
RegisterBuiltins(reg)
in, err := reg.BuildInput(sourceConfigForDriver("nws_weatherstories"))
if err != nil {
t.Fatalf("BuildInput(nws_weatherstories) error = %v", err)
}
if _, ok := in.(fksource.PollSource); !ok {
t.Fatalf("BuildInput(nws_weatherstories) type = %T, want PollSource", in)
for _, tt := range pollDriverRegistrations {
tt := tt
t.Run(tt.driver, func(t *testing.T) {
in, err := reg.BuildInput(sourceConfigForDriver(tt.driver))
if err != nil {
t.Fatalf("BuildInput(%q) error = %v", tt.driver, err)
}
if _, ok := in.(fksource.PollSource); !ok {
t.Fatalf("BuildInput(%q) type = %T, want PollSource", tt.driver, in)
}
})
}
}
@@ -73,44 +45,16 @@ func TestRegisterBuiltinsDoesNotRegisterLegacyNWSForecastDriver(t *testing.T) {
}
}
func TestRegisterBuiltinsRegistersAllCurrentDrivers(t *testing.T) {
reg := fksource.NewRegistry()
RegisterBuiltins(reg)
drivers := []string{
"nws_observation",
"nws_alerts",
"nws_forecast_hourly",
"nws_forecast_narrative",
"nws_forecast_discussion",
"nws_weatherstories",
"openmeteo_observation",
"openmeteo_forecast",
"openweather_observation",
"spc_convective_outlook",
}
for _, driver := range drivers {
in, err := reg.BuildInput(sourceConfigForDriver(driver))
if err != nil {
t.Fatalf("BuildInput(%s) error = %v", driver, err)
}
if _, ok := in.(fksource.PollSource); !ok {
t.Fatalf("BuildInput(%s) type = %T, want PollSource", driver, in)
}
}
}
func sourceConfigForDriver(driver string) config.SourceConfig {
url := "https://example.invalid"
if driver == "openweather_observation" {
if driver == openweather.DriverObservation {
url = "https://example.invalid?units=metric"
}
params := map[string]any{
"url": url,
"user_agent": "test-agent",
}
if driver == "spc_convective_outlook" {
if driver == spc.DriverConvectiveOutlook {
params["latitude"] = 38.6239
params["longitude"] = -90.3571
}

View File

@@ -0,0 +1,32 @@
package sources
import (
"os"
"strings"
"testing"
)
func TestDocumentedRegisteredSourceDrivers(t *testing.T) {
docs := map[string]string{
"docs/config.md": readDoc(t, "../../docs/config.md"),
"docs/internal/sources.md": readDoc(t, "../../docs/internal/sources.md"),
}
for _, reg := range pollDriverRegistrations {
for path, doc := range docs {
if !strings.Contains(doc, reg.driver) {
t.Fatalf("%s missing source driver %q", path, reg.driver)
}
}
}
}
func readDoc(t *testing.T, path string) string {
t.Helper()
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("ReadFile(%s) error = %v", path, err)
}
return string(raw)
}

View File

@@ -0,0 +1,58 @@
package httpconfig
import (
"fmt"
"strings"
"time"
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/transport"
)
// Settings contains common HTTP client config for sources that fetch multiple documents.
type Settings struct {
Name string
UserAgent string
Timeout time.Duration
BodyLimitBytes int64
}
func Parse(driver string, cfg config.SourceConfig) (Settings, error) {
name := strings.TrimSpace(cfg.Name)
if name == "" {
return Settings{}, fmt.Errorf("%s: name is required", driver)
}
if cfg.Params == nil {
return Settings{}, fmt.Errorf("%s %q: params are required", driver, name)
}
userAgent, ok := cfg.ParamString("user_agent", "userAgent")
if !ok {
return Settings{}, fmt.Errorf("%s %q: params.user_agent is required", driver, name)
}
timeout := transport.DefaultHTTPTimeout
if _, exists := cfg.Params["http_timeout"]; exists {
var ok bool
timeout, ok = cfg.ParamDuration("http_timeout")
if !ok || timeout <= 0 {
return Settings{}, fmt.Errorf("source %q: params.http_timeout must be a positive duration", name)
}
}
bodyLimit := transport.DefaultHTTPResponseBodyLimitBytes
if _, exists := cfg.Params["http_response_body_limit_bytes"]; exists {
rawLimit, ok := cfg.ParamInt("http_response_body_limit_bytes")
if !ok || rawLimit <= 0 {
return Settings{}, fmt.Errorf("source %q: params.http_response_body_limit_bytes must be a positive integer", name)
}
bodyLimit = int64(rawLimit)
}
return Settings{
Name: name,
UserAgent: userAgent,
Timeout: timeout,
BodyLimitBytes: bodyLimit,
}, nil
}

View File

@@ -0,0 +1,115 @@
package httpconfig
import (
"strings"
"testing"
"time"
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/transport"
)
func TestParseUsesRequiredValuesAndDefaults(t *testing.T) {
got, err := Parse("test_driver", config.SourceConfig{
Name: " test-source ",
Params: map[string]any{
"user_agent": "test-agent",
},
})
if err != nil {
t.Fatalf("Parse() error = %v", err)
}
if got.Name != "test-source" {
t.Fatalf("Name = %q, want test-source", got.Name)
}
if got.UserAgent != "test-agent" {
t.Fatalf("UserAgent = %q, want test-agent", got.UserAgent)
}
if got.Timeout != transport.DefaultHTTPTimeout {
t.Fatalf("Timeout = %s, want %s", got.Timeout, transport.DefaultHTTPTimeout)
}
if got.BodyLimitBytes != transport.DefaultHTTPResponseBodyLimitBytes {
t.Fatalf("BodyLimitBytes = %d, want %d", got.BodyLimitBytes, transport.DefaultHTTPResponseBodyLimitBytes)
}
}
func TestParseUsesAliasesAndOverrides(t *testing.T) {
got, err := Parse("test_driver", config.SourceConfig{
Name: "test-source",
Params: map[string]any{
"userAgent": "test-agent",
"http_timeout": "2s",
"http_response_body_limit_bytes": 2048,
},
})
if err != nil {
t.Fatalf("Parse() error = %v", err)
}
if got.UserAgent != "test-agent" {
t.Fatalf("UserAgent = %q, want test-agent", got.UserAgent)
}
if got.Timeout != 2*time.Second {
t.Fatalf("Timeout = %s, want 2s", got.Timeout)
}
if got.BodyLimitBytes != 2048 {
t.Fatalf("BodyLimitBytes = %d, want 2048", got.BodyLimitBytes)
}
}
func TestParseRejectsInvalidConfig(t *testing.T) {
tests := []struct {
name string
cfg config.SourceConfig
wantErr string
}{
{
name: "missing name",
cfg: config.SourceConfig{Params: map[string]any{"user_agent": "test-agent"}},
wantErr: "test_driver: name is required",
},
{
name: "missing params",
cfg: config.SourceConfig{Name: "test-source"},
wantErr: `test_driver "test-source": params are required`,
},
{
name: "missing user agent",
cfg: config.SourceConfig{Name: "test-source", Params: map[string]any{}},
wantErr: `test_driver "test-source": params.user_agent is required`,
},
{
name: "invalid timeout",
cfg: config.SourceConfig{
Name: "test-source",
Params: map[string]any{
"user_agent": "test-agent",
"http_timeout": "0s",
},
},
wantErr: `source "test-source": params.http_timeout must be a positive duration`,
},
{
name: "invalid body limit",
cfg: config.SourceConfig{
Name: "test-source",
Params: map[string]any{
"user_agent": "test-agent",
"http_response_body_limit_bytes": 0,
},
},
wantErr: `source "test-source": params.http_response_body_limit_bytes must be a positive integer`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := Parse("test_driver", tt.cfg)
if err == nil {
t.Fatalf("Parse() error = nil, want %q", tt.wantErr)
}
if !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("Parse() error = %q, want %q", err, tt.wantErr)
}
})
}
}

View File

@@ -26,10 +26,8 @@ type AlertsSource struct {
}
func NewAlertsSource(cfg config.SourceConfig) (*AlertsSource, error) {
const driver = "nws_alerts"
// NWS alerts responses are GeoJSON-ish; allow fallback to plain JSON as well.
hs, err := fksources.NewHTTPSource(driver, cfg, "application/geo+json, application/json")
hs, err := fksources.NewHTTPSource(DriverAlerts, cfg, "application/geo+json, application/json")
if err != nil {
return nil, err
}
@@ -40,7 +38,7 @@ func NewAlertsSource(cfg config.SourceConfig) (*AlertsSource, error) {
func (s *AlertsSource) Name() string { return s.http.Name }
// Kinds is used for routing/policy.
func (s *AlertsSource) Kinds() []event.Kind { return []event.Kind{event.Kind("alert")} }
func (s *AlertsSource) Kinds() []event.Kind { return []event.Kind{event.Kind(standards.KindAlert)} }
func (s *AlertsSource) Poll(ctx context.Context) ([]event.Event, error) {
raw, meta, changed, err := s.fetchRaw(ctx)
@@ -71,7 +69,7 @@ func (s *AlertsSource) Poll(ctx context.Context) ([]event.Event, error) {
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("alert"),
event.Kind(standards.KindAlert),
s.http.Name,
standards.SchemaRawNWSAlertsV1,
eventID,

View File

@@ -0,0 +1,11 @@
package nws
// Source driver strings registered by weatherfeeder for NWS sources.
const (
DriverObservation = "nws_observation"
DriverAlerts = "nws_alerts"
DriverForecastHourly = "nws_forecast_hourly"
DriverForecastNarrative = "nws_forecast_narrative"
DriverForecastDiscussion = "nws_forecast_discussion"
DriverWeatherStories = "nws_weatherstories"
)

View File

@@ -10,6 +10,7 @@ import (
"gitea.maximumdirect.net/ejr/feedkit/event"
fksources "gitea.maximumdirect.net/ejr/feedkit/sources"
nwscommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/nws"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
const nwsForecastAccept = "application/geo+json, application/json"
@@ -44,7 +45,9 @@ func newForecastSource(cfg config.SourceConfig, driver, rawSchema string) (*fore
func (s *forecastSource) Name() string { return s.http.Name }
func (s *forecastSource) Kinds() []event.Kind { return []event.Kind{event.Kind("forecast")} }
func (s *forecastSource) Kinds() []event.Kind {
return []event.Kind{event.Kind(standards.KindForecast)}
}
func (s *forecastSource) Poll(ctx context.Context) ([]event.Event, error) {
raw, meta, changed, err := s.fetchRaw(ctx)
@@ -69,7 +72,7 @@ func (s *forecastSource) Poll(ctx context.Context) ([]event.Event, error) {
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("forecast"),
event.Kind(standards.KindForecast),
s.http.Name,
s.rawSchema,
eventID,

View File

@@ -20,9 +20,7 @@ type ForecastDiscussionSource struct {
}
func NewForecastDiscussionSource(cfg config.SourceConfig) (*ForecastDiscussionSource, error) {
const driver = "nws_forecast_discussion"
hs, err := fksources.NewHTTPSource(driver, cfg, "text/html, application/xhtml+xml")
hs, err := fksources.NewHTTPSource(DriverForecastDiscussion, cfg, "text/html, application/xhtml+xml")
if err != nil {
return nil, err
}
@@ -33,7 +31,7 @@ func NewForecastDiscussionSource(cfg config.SourceConfig) (*ForecastDiscussionSo
func (s *ForecastDiscussionSource) Name() string { return s.http.Name }
func (s *ForecastDiscussionSource) Kinds() []event.Kind {
return []event.Kind{event.Kind("forecast_discussion")}
return []event.Kind{event.Kind(standards.KindForecastDiscussion)}
}
func (s *ForecastDiscussionSource) Poll(ctx context.Context) ([]event.Event, error) {
@@ -57,7 +55,7 @@ func (s *ForecastDiscussionSource) Poll(ctx context.Context) ([]event.Event, err
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("forecast_discussion"),
event.Kind(standards.KindForecastDiscussion),
s.http.Name,
standards.SchemaRawNWSForecastDiscussionV1,
eventID,

View File

@@ -27,7 +27,7 @@ func TestForecastDiscussionSourcePollEmitsExpectedEvent(t *testing.T) {
if err != nil {
t.Fatalf("NewForecastDiscussionSource() error = %v", err)
}
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("forecast_discussion") {
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindForecastDiscussion) {
t.Fatalf("Kinds() = %#v, want [forecast_discussion]", got)
}
@@ -40,7 +40,7 @@ func TestForecastDiscussionSourcePollEmitsExpectedEvent(t *testing.T) {
}
got := events[0]
if got.Kind != event.Kind("forecast_discussion") {
if got.Kind != event.Kind(standards.KindForecastDiscussion) {
t.Fatalf("Kind = %q, want forecast_discussion", got.Kind)
}
if got.Schema != standards.SchemaRawNWSForecastDiscussionV1 {
@@ -117,7 +117,7 @@ func TestForecastDiscussionSourcePollRejectsInvalidHTML(t *testing.T) {
func forecastDiscussionSourceConfig(url string) config.SourceConfig {
return config.SourceConfig{
Name: "test-forecast-discussion-source",
Driver: "nws_forecast_discussion",
Driver: DriverForecastDiscussion,
Mode: config.SourceModePoll,
Params: map[string]any{
"url": url,

View File

@@ -18,8 +18,7 @@ type HourlyForecastSource struct {
}
func NewHourlyForecastSource(cfg config.SourceConfig) (*HourlyForecastSource, error) {
const driver = "nws_forecast_hourly"
src, err := newForecastSource(cfg, driver, standards.SchemaRawNWSHourlyForecastV1)
src, err := newForecastSource(cfg, DriverForecastHourly, standards.SchemaRawNWSHourlyForecastV1)
if err != nil {
return nil, err
}

View File

@@ -18,8 +18,7 @@ type NarrativeForecastSource struct {
}
func NewNarrativeForecastSource(cfg config.SourceConfig) (*NarrativeForecastSource, error) {
const driver = "nws_forecast_narrative"
src, err := newForecastSource(cfg, driver, standards.SchemaRawNWSNarrativeForecastV1)
src, err := newForecastSource(cfg, DriverForecastNarrative, standards.SchemaRawNWSNarrativeForecastV1)
if err != nil {
return nil, err
}

View File

@@ -26,7 +26,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
}{
{
name: "hourly",
driver: "nws_forecast_hourly",
driver: DriverForecastHourly,
wantSchema: standards.SchemaRawNWSHourlyForecastV1,
newSource: func(cfg config.SourceConfig) (forecastPoller, error) {
return NewHourlyForecastSource(cfg)
@@ -34,7 +34,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
},
{
name: "narrative",
driver: "nws_forecast_narrative",
driver: DriverForecastNarrative,
wantSchema: standards.SchemaRawNWSNarrativeForecastV1,
newSource: func(cfg config.SourceConfig) (forecastPoller, error) {
return NewNarrativeForecastSource(cfg)
@@ -55,7 +55,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
}
if ks, ok := src.(interface{ Kinds() []event.Kind }); !ok {
t.Fatalf("source does not implement Kinds()")
} else if gotKinds := ks.Kinds(); len(gotKinds) != 1 || gotKinds[0] != event.Kind("forecast") {
} else if gotKinds := ks.Kinds(); len(gotKinds) != 1 || gotKinds[0] != event.Kind(standards.KindForecast) {
t.Fatalf("Kinds() = %#v, want [forecast]", gotKinds)
}
@@ -69,7 +69,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
if got[0].Schema != tt.wantSchema {
t.Fatalf("Poll() schema = %q, want %q", got[0].Schema, tt.wantSchema)
}
if got[0].Kind != event.Kind("forecast") {
if got[0].Kind != event.Kind(standards.KindForecast) {
t.Fatalf("Poll() kind = %q, want forecast", got[0].Kind)
}
@@ -117,7 +117,7 @@ func TestForecastSourcePollEffectiveAtFallbackOrder(t *testing.T) {
}))
defer srv.Close()
src, err := NewHourlyForecastSource(forecastSourceConfig("nws_forecast_hourly", srv.URL))
src, err := NewHourlyForecastSource(forecastSourceConfig(DriverForecastHourly, srv.URL))
if err != nil {
t.Fatalf("NewHourlyForecastSource() error = %v", err)
}
@@ -148,7 +148,7 @@ func TestForecastSourcePollMetadataDecodeFailureStillEmitsRawEvent(t *testing.T)
}))
defer srv.Close()
src, err := NewNarrativeForecastSource(forecastSourceConfig("nws_forecast_narrative", srv.URL))
src, err := NewNarrativeForecastSource(forecastSourceConfig(DriverForecastNarrative, srv.URL))
if err != nil {
t.Fatalf("NewNarrativeForecastSource() error = %v", err)
}

View File

@@ -20,9 +20,7 @@ type ObservationSource struct {
}
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
const driver = "nws_observation"
hs, err := fksources.NewHTTPSource(driver, cfg, "application/geo+json, application/json")
hs, err := fksources.NewHTTPSource(DriverObservation, cfg, "application/geo+json, application/json")
if err != nil {
return nil, err
}
@@ -32,7 +30,9 @@ func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
func (s *ObservationSource) Name() string { return s.http.Name }
func (s *ObservationSource) Kinds() []event.Kind { return []event.Kind{event.Kind("observation")} }
func (s *ObservationSource) Kinds() []event.Kind {
return []event.Kind{event.Kind(standards.KindObservation)}
}
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
raw, meta, changed, err := s.fetchRaw(ctx)
@@ -54,7 +54,7 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
eventID := fksources.DefaultEventID(meta.ID, s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("observation"),
event.Kind(standards.KindObservation),
s.http.Name,
standards.SchemaRawNWSObservationV1,
eventID,

View File

@@ -8,6 +8,7 @@ import (
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
@@ -31,7 +32,7 @@ func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
src, err := NewObservationSource(config.SourceConfig{
Name: "NWSObservationTest",
Driver: "nws_observation",
Driver: DriverObservation,
Mode: config.SourceModePoll,
Params: map[string]any{
"url": srv.URL,
@@ -41,7 +42,7 @@ func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
if err != nil {
t.Fatalf("NewObservationSource() error = %v", err)
}
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("observation") {
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindObservation) {
t.Fatalf("Kinds() = %#v, want [observation]", got)
}
@@ -52,7 +53,7 @@ func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
if len(first) != 1 {
t.Fatalf("first Poll() len = %d, want 1", len(first))
}
if first[0].Kind != event.Kind("observation") {
if first[0].Kind != event.Kind(standards.KindObservation) {
t.Fatalf("first Poll() kind = %q", first[0].Kind)
}

View File

@@ -22,9 +22,7 @@ type WeatherStoriesSource struct {
}
func NewWeatherStoriesSource(cfg config.SourceConfig) (*WeatherStoriesSource, error) {
const driver = "nws_weatherstories"
hs, err := fksources.NewHTTPSource(driver, cfg, "application/geo+json, application/json")
hs, err := fksources.NewHTTPSource(DriverWeatherStories, cfg, "application/geo+json, application/json")
if err != nil {
return nil, err
}
@@ -35,7 +33,7 @@ func NewWeatherStoriesSource(cfg config.SourceConfig) (*WeatherStoriesSource, er
func (s *WeatherStoriesSource) Name() string { return s.http.Name }
func (s *WeatherStoriesSource) Kinds() []event.Kind {
return []event.Kind{event.Kind("weather_story")}
return []event.Kind{event.Kind(standards.KindWeatherStory)}
}
func (s *WeatherStoriesSource) Poll(ctx context.Context) ([]event.Event, error) {
@@ -61,7 +59,7 @@ func (s *WeatherStoriesSource) Poll(ctx context.Context) ([]event.Event, error)
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("weather_story"),
event.Kind(standards.KindWeatherStory),
s.http.Name,
standards.SchemaRawNWSWeatherStoriesV1,
eventID,

View File

@@ -28,7 +28,7 @@ func TestWeatherStoriesSourcePollEmitsExpectedEventAndPrefersLatestUpdateTime(t
if err != nil {
t.Fatalf("NewWeatherStoriesSource() error = %v", err)
}
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("weather_story") {
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindWeatherStory) {
t.Fatalf("Kinds() = %#v, want [weather_story]", got)
}
@@ -41,7 +41,7 @@ func TestWeatherStoriesSourcePollEmitsExpectedEventAndPrefersLatestUpdateTime(t
}
got := events[0]
if got.Kind != event.Kind("weather_story") {
if got.Kind != event.Kind(standards.KindWeatherStory) {
t.Fatalf("Kind = %q, want weather_story", got.Kind)
}
if got.Schema != standards.SchemaRawNWSWeatherStoriesV1 {
@@ -148,7 +148,7 @@ func TestWeatherStoriesSourcePollMetadataDecodeFailureStillEmitsRawEvent(t *test
func weatherStoriesSourceConfig(url string) config.SourceConfig {
return config.SourceConfig{
Name: "test-weatherstories-source",
Driver: "nws_weatherstories",
Driver: DriverWeatherStories,
Mode: config.SourceModePoll,
Params: map[string]any{
"url": url,

View File

@@ -0,0 +1,7 @@
package openmeteo
// Source driver strings registered by weatherfeeder for Open-Meteo sources.
const (
DriverObservation = "openmeteo_observation"
DriverForecast = "openmeteo_forecast"
)

View File

@@ -19,9 +19,7 @@ type ForecastSource struct {
}
func NewForecastSource(cfg config.SourceConfig) (*ForecastSource, error) {
const driver = "openmeteo_forecast"
hs, err := fksources.NewHTTPSource(driver, cfg, "application/json")
hs, err := fksources.NewHTTPSource(DriverForecast, cfg, "application/json")
if err != nil {
return nil, err
}
@@ -31,7 +29,9 @@ func NewForecastSource(cfg config.SourceConfig) (*ForecastSource, error) {
func (s *ForecastSource) Name() string { return s.http.Name }
func (s *ForecastSource) Kinds() []event.Kind { return []event.Kind{event.Kind("forecast")} }
func (s *ForecastSource) Kinds() []event.Kind {
return []event.Kind{event.Kind(standards.KindForecast)}
}
func (s *ForecastSource) Poll(ctx context.Context) ([]event.Event, error) {
raw, meta, changed, err := s.fetchRaw(ctx)
@@ -55,7 +55,7 @@ func (s *ForecastSource) Poll(ctx context.Context) ([]event.Event, error) {
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("forecast"),
event.Kind(standards.KindForecast),
s.http.Name,
standards.SchemaRawOpenMeteoHourlyForecastV1,
eventID,

View File

@@ -19,9 +19,7 @@ type ObservationSource struct {
}
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
const driver = "openmeteo_observation"
hs, err := fksources.NewHTTPSource(driver, cfg, "application/json")
hs, err := fksources.NewHTTPSource(DriverObservation, cfg, "application/json")
if err != nil {
return nil, err
}
@@ -31,7 +29,9 @@ func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
func (s *ObservationSource) Name() string { return s.http.Name }
func (s *ObservationSource) Kinds() []event.Kind { return []event.Kind{event.Kind("observation")} }
func (s *ObservationSource) Kinds() []event.Kind {
return []event.Kind{event.Kind(standards.KindObservation)}
}
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
raw, meta, changed, err := s.fetchRaw(ctx)
@@ -52,7 +52,7 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("observation"),
event.Kind(standards.KindObservation),
s.http.Name,
standards.SchemaRawOpenMeteoCurrentV1,
eventID,

View File

@@ -5,12 +5,13 @@ import (
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
func TestObservationSourceAdvertisesKinds(t *testing.T) {
src, err := NewObservationSource(config.SourceConfig{
Name: "openmeteo-observation-test",
Driver: "openmeteo_observation",
Driver: DriverObservation,
Mode: config.SourceModePoll,
Params: map[string]any{
"url": "https://example.invalid",
@@ -20,7 +21,7 @@ func TestObservationSourceAdvertisesKinds(t *testing.T) {
if err != nil {
t.Fatalf("NewObservationSource() error = %v", err)
}
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("observation") {
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindObservation) {
t.Fatalf("Kinds() = %#v, want [observation]", got)
}
}
@@ -28,7 +29,7 @@ func TestObservationSourceAdvertisesKinds(t *testing.T) {
func TestForecastSourceAdvertisesKinds(t *testing.T) {
src, err := NewForecastSource(config.SourceConfig{
Name: "openmeteo-forecast-test",
Driver: "openmeteo_forecast",
Driver: DriverForecast,
Mode: config.SourceModePoll,
Params: map[string]any{
"url": "https://example.invalid",
@@ -38,7 +39,7 @@ func TestForecastSourceAdvertisesKinds(t *testing.T) {
if err != nil {
t.Fatalf("NewForecastSource() error = %v", err)
}
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("forecast") {
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindForecast) {
t.Fatalf("Kinds() = %#v, want [forecast]", got)
}
}

View File

@@ -0,0 +1,6 @@
package openweather
// Source driver strings registered by weatherfeeder for OpenWeather sources.
const (
DriverObservation = "openweather_observation"
)

View File

@@ -19,9 +19,7 @@ type ObservationSource struct {
}
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
const driver = "openweather_observation"
hs, err := fksources.NewHTTPSource(driver, cfg, "application/json")
hs, err := fksources.NewHTTPSource(DriverObservation, cfg, "application/json")
if err != nil {
return nil, err
}
@@ -35,7 +33,9 @@ func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
func (s *ObservationSource) Name() string { return s.http.Name }
func (s *ObservationSource) Kinds() []event.Kind { return []event.Kind{event.Kind("observation")} }
func (s *ObservationSource) Kinds() []event.Kind {
return []event.Kind{event.Kind(standards.KindObservation)}
}
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
if err := owcommon.RequireMetricUnits(s.http.URL); err != nil {
@@ -60,7 +60,7 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("observation"),
event.Kind(standards.KindObservation),
s.http.Name,
standards.SchemaRawOpenWeatherCurrentV1,
eventID,

View File

@@ -5,12 +5,13 @@ import (
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
func TestObservationSourceAdvertisesKinds(t *testing.T) {
src, err := NewObservationSource(config.SourceConfig{
Name: "openweather-observation-test",
Driver: "openweather_observation",
Driver: DriverObservation,
Mode: config.SourceModePoll,
Params: map[string]any{
"url": "https://example.invalid?units=metric",
@@ -20,7 +21,7 @@ func TestObservationSourceAdvertisesKinds(t *testing.T) {
if err != nil {
t.Fatalf("NewObservationSource() error = %v", err)
}
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("observation") {
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindObservation) {
t.Fatalf("Kinds() = %#v, want [observation]", got)
}
}

View File

@@ -16,12 +16,11 @@ import (
fksources "gitea.maximumdirect.net/ejr/feedkit/sources"
"gitea.maximumdirect.net/ejr/feedkit/transport"
spcprovider "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/spc"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/internal/httpconfig"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
)
const (
driverConvectiveOutlook = "spc_convective_outlook"
acceptGeoJSON = "application/geo+json, application/json"
acceptDiscussion = "text/html, application/xhtml+xml"
acceptRSS = "application/rss+xml, application/xml, text/xml"
@@ -56,53 +55,27 @@ type ConvectiveOutlookSource struct {
}
func NewConvectiveOutlookSource(cfg config.SourceConfig) (*ConvectiveOutlookSource, error) {
name := strings.TrimSpace(cfg.Name)
if name == "" {
return nil, fmt.Errorf("%s: name is required", driverConvectiveOutlook)
}
if cfg.Params == nil {
return nil, fmt.Errorf("%s %q: params are required", driverConvectiveOutlook, name)
}
userAgent, ok := cfg.ParamString("user_agent", "userAgent")
if !ok {
return nil, fmt.Errorf("%s %q: params.user_agent is required", driverConvectiveOutlook, name)
httpSettings, err := httpconfig.Parse(DriverConvectiveOutlook, cfg)
if err != nil {
return nil, err
}
latitude, err := requireFloatParam(cfg, "latitude")
if err != nil {
return nil, fmt.Errorf("%s %q: %w", driverConvectiveOutlook, name, err)
return nil, fmt.Errorf("%s %q: %w", DriverConvectiveOutlook, httpSettings.Name, err)
}
longitude, err := requireFloatParam(cfg, "longitude")
if err != nil {
return nil, fmt.Errorf("%s %q: %w", driverConvectiveOutlook, name, err)
}
timeout := transport.DefaultHTTPTimeout
if _, exists := cfg.Params["http_timeout"]; exists {
var ok bool
timeout, ok = cfg.ParamDuration("http_timeout")
if !ok || timeout <= 0 {
return nil, fmt.Errorf("source %q: params.http_timeout must be a positive duration", name)
}
}
bodyLimit := transport.DefaultHTTPResponseBodyLimitBytes
if _, exists := cfg.Params["http_response_body_limit_bytes"]; exists {
rawLimit, ok := cfg.ParamInt("http_response_body_limit_bytes")
if !ok || rawLimit <= 0 {
return nil, fmt.Errorf("source %q: params.http_response_body_limit_bytes must be a positive integer", name)
}
bodyLimit = int64(rawLimit)
return nil, fmt.Errorf("%s %q: %w", DriverConvectiveOutlook, httpSettings.Name, err)
}
geoJSONProducts, err := configuredGeoJSONProducts(cfg)
if err != nil {
return nil, fmt.Errorf("%s %q: %w", driverConvectiveOutlook, name, err)
return nil, fmt.Errorf("%s %q: %w", DriverConvectiveOutlook, httpSettings.Name, err)
}
discussions, err := configuredDiscussionProducts(cfg)
if err != nil {
return nil, fmt.Errorf("%s %q: %w", driverConvectiveOutlook, name, err)
return nil, fmt.Errorf("%s %q: %w", DriverConvectiveOutlook, httpSettings.Name, err)
}
rssURL := ""
@@ -114,14 +87,14 @@ func NewConvectiveOutlookSource(cfg config.SourceConfig) (*ConvectiveOutlookSour
locationName, _ := cfg.ParamString("location_name", "locationName")
return &ConvectiveOutlookSource{
name: name,
userAgent: userAgent,
name: httpSettings.Name,
userAgent: httpSettings.UserAgent,
locationID: locationID,
locationName: locationName,
latitude: latitude,
longitude: longitude,
client: transport.NewHTTPClient(timeout),
bodyLimit: bodyLimit,
client: transport.NewHTTPClient(httpSettings.Timeout),
bodyLimit: httpSettings.BodyLimitBytes,
geoJSONProducts: geoJSONProducts,
discussions: discussions,
rssURL: rssURL,
@@ -131,7 +104,7 @@ func NewConvectiveOutlookSource(cfg config.SourceConfig) (*ConvectiveOutlookSour
func (s *ConvectiveOutlookSource) Name() string { return s.name }
func (s *ConvectiveOutlookSource) Kinds() []event.Kind {
return []event.Kind{event.Kind("outlook")}
return []event.Kind{event.Kind(standards.KindOutlook)}
}
func (s *ConvectiveOutlookSource) Poll(ctx context.Context) ([]event.Event, error) {
@@ -221,7 +194,7 @@ func (s *ConvectiveOutlookSource) Poll(ctx context.Context) ([]event.Event, erro
eventID := fksources.DefaultEventID("", s.name, &effectiveAt, emittedAt)
return fksources.SingleEvent(
event.Kind("outlook"),
event.Kind(standards.KindOutlook),
s.name,
standards.SchemaRawSPCConvectiveOutlookV1,
eventID,

View File

@@ -22,7 +22,7 @@ func TestConvectiveOutlookSourceKinds(t *testing.T) {
t.Fatalf("NewConvectiveOutlookSource() error = %v", err)
}
got := src.Kinds()
if len(got) != 1 || got[0] != event.Kind("outlook") {
if len(got) != 1 || got[0] != event.Kind(standards.KindOutlook) {
t.Fatalf("Kinds() = %#v, want [outlook]", got)
}
}
@@ -57,7 +57,7 @@ func TestConvectiveOutlookSourcePollEmitsRawBundle(t *testing.T) {
t.Fatalf("Poll() returned %d events, want 1", len(events))
}
got := events[0]
if got.Kind != event.Kind("outlook") {
if got.Kind != event.Kind(standards.KindOutlook) {
t.Fatalf("Kind = %q, want outlook", got.Kind)
}
if got.Schema != standards.SchemaRawSPCConvectiveOutlookV1 {
@@ -277,7 +277,7 @@ func convectiveOutlookConfig(extra map[string]any) config.SourceConfig {
}
return config.SourceConfig{
Name: "spc-test",
Driver: driverConvectiveOutlook,
Driver: DriverConvectiveOutlook,
Mode: config.SourceModePoll,
Params: params,
}

View File

@@ -0,0 +1,6 @@
package spc
// Source driver strings registered by weatherfeeder for SPC sources.
const (
DriverConvectiveOutlook = "spc_convective_outlook"
)

69
standards/docs_test.go Normal file
View File

@@ -0,0 +1,69 @@
package standards
import (
"go/ast"
"go/parser"
"go/token"
"os"
"strconv"
"strings"
"testing"
)
func TestDocumentedEventSchemas(t *testing.T) {
raw, err := os.ReadFile("../docs/integrations/events.md")
if err != nil {
t.Fatalf("ReadFile(events.md) error = %v", err)
}
doc := string(raw)
schemas := schemaConstants(t)
for _, schema := range schemas {
if !strings.Contains(doc, schema) {
t.Fatalf("docs/integrations/events.md missing schema %q", schema)
}
}
}
func schemaConstants(t *testing.T) []string {
t.Helper()
file, err := parser.ParseFile(token.NewFileSet(), "schema.go", nil, 0)
if err != nil {
t.Fatalf("ParseFile(schema.go) error = %v", err)
}
var out []string
ast.Inspect(file, func(n ast.Node) bool {
valueSpec, ok := n.(*ast.ValueSpec)
if !ok {
return true
}
for i, name := range valueSpec.Names {
if !strings.HasPrefix(name.Name, "Schema") || schemaConstantNotInCurrentContract(name.Name) {
continue
}
if i >= len(valueSpec.Values) {
t.Fatalf("schema constant %s has no explicit value", name.Name)
}
lit, ok := valueSpec.Values[i].(*ast.BasicLit)
if !ok || lit.Kind != token.STRING {
t.Fatalf("schema constant %s is not a string literal", name.Name)
}
schema, err := strconv.Unquote(lit.Value)
if err != nil {
t.Fatalf("schema constant %s value is not a quoted string: %v", name.Name, err)
}
out = append(out, schema)
}
return true
})
if len(out) == 0 {
t.Fatalf("no schema constants found")
}
return out
}
func schemaConstantNotInCurrentContract(name string) bool {
return name == "SchemaRawOpenWeatherHourlyForecastV1"
}

11
standards/kind.go Normal file
View File

@@ -0,0 +1,11 @@
package standards
// Event kind strings used by weatherfeeder events and routing policy.
const (
KindObservation = "observation"
KindForecast = "forecast"
KindForecastDiscussion = "forecast_discussion"
KindWeatherStory = "weather_story"
KindAlert = "alert"
KindOutlook = "outlook"
)