7 Commits

Author SHA1 Message Date
06d5973746 Clean up stale internal literals
Some checks failed
ci/woodpecker/push/build-image Pipeline was canceled
2026-06-11 02:23:42 +00:00
8045b27173 Move SPC provider fixture helper 2026-06-11 02:20:25 +00:00
985468c1b9 Add documentation identifier consistency tests 2026-06-11 02:18:15 +00:00
6a0b30b7c7 Centralize Postgres event envelope mapping 2026-06-11 02:15:52 +00:00
86ce4eb68c Share HTTP config parsing for multi-document sources 2026-06-11 02:13:38 +00:00
33541a71fc Table-drive source registry tests 2026-06-11 02:10:13 +00:00
b7277e0c02 Centralize weather event and driver identifiers 2026-06-11 02:08:36 +00:00
40 changed files with 528 additions and 309 deletions

View File

@@ -20,6 +20,7 @@ import (
wfnormalizers "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers" wfnormalizers "gitea.maximumdirect.net/ejr/weatherfeeder/internal/normalizers"
wfsources "gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources" wfsources "gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
) )
type testInput struct { type testInput struct {
@@ -36,10 +37,10 @@ type testKindsSource struct {
func (s testKindsSource) Kinds() []fkevent.Kind { return s.kinds } func (s testKindsSource) Kinds() []fkevent.Kind { return s.kinds }
func TestValidateSourceExpectedKindsSubsetAllowed(t *testing.T) { func TestValidateSourceExpectedKindsSubsetAllowed(t *testing.T) {
sc := config.SourceConfig{Kinds: []string{"observation"}} sc := config.SourceConfig{Kinds: []string{standards.KindObservation}}
in := testKindsSource{ in := testKindsSource{
testInput: testInput{name: "test"}, testInput: testInput{name: "test"},
kinds: []fkevent.Kind{"observation", "forecast"}, kinds: []fkevent.Kind{fkevent.Kind(standards.KindObservation), fkevent.Kind(standards.KindForecast)},
} }
if err := fksources.ValidateExpectedKinds(sc, in); err != nil { if err := fksources.ValidateExpectedKinds(sc, in); err != nil {
@@ -48,10 +49,10 @@ func TestValidateSourceExpectedKindsSubsetAllowed(t *testing.T) {
} }
func TestValidateSourceExpectedKindsMismatchFails(t *testing.T) { func TestValidateSourceExpectedKindsMismatchFails(t *testing.T) {
sc := config.SourceConfig{Kinds: []string{"alert"}} sc := config.SourceConfig{Kinds: []string{standards.KindAlert}}
in := testKindsSource{ in := testKindsSource{
testInput: testInput{name: "test"}, testInput: testInput{name: "test"},
kinds: []fkevent.Kind{"observation", "forecast"}, kinds: []fkevent.Kind{fkevent.Kind(standards.KindObservation), fkevent.Kind(standards.KindForecast)},
} }
err := fksources.ValidateExpectedKinds(sc, in) err := fksources.ValidateExpectedKinds(sc, in)
@@ -64,7 +65,7 @@ func TestValidateSourceExpectedKindsMismatchFails(t *testing.T) {
} }
func TestValidateSourceExpectedKindsNoMetadataSkipsCheck(t *testing.T) { func TestValidateSourceExpectedKindsNoMetadataSkipsCheck(t *testing.T) {
sc := config.SourceConfig{Kinds: []string{"alert"}} sc := config.SourceConfig{Kinds: []string{standards.KindAlert}}
in := testInput{name: "test"} in := testInput{name: "test"}
if err := fksources.ValidateExpectedKinds(sc, in); err != nil { if err := fksources.ValidateExpectedKinds(sc, in); err != nil {
@@ -111,6 +112,10 @@ func TestMaintainedConfigExamplesLoad(t *testing.T) {
func assertConfigSourcesBuildSchedulerJobs(t *testing.T, cfg *config.Config) { func assertConfigSourcesBuildSchedulerJobs(t *testing.T, cfg *config.Config) {
t.Helper() t.Helper()
if len(cfg.Sources) == 0 {
t.Fatalf("config has no sources")
}
reg := fksources.NewRegistry() reg := fksources.NewRegistry()
wfsources.RegisterBuiltins(reg) wfsources.RegisterBuiltins(reg)
@@ -158,7 +163,7 @@ func TestNormalizeNoMatchPassThrough(t *testing.T) {
pl := &fkpipeline.Pipeline{Processors: chain} pl := &fkpipeline.Pipeline{Processors: chain}
in := fkevent.Event{ in := fkevent.Event{
ID: "evt-no-match", ID: "evt-no-match",
Kind: fkevent.Kind("observation"), Kind: fkevent.Kind(standards.KindObservation),
Source: "test", Source: "test",
EmittedAt: time.Now().UTC(), EmittedAt: time.Now().UTC(),
Schema: "raw.weatherfeeder.unknown.v1", Schema: "raw.weatherfeeder.unknown.v1",
@@ -188,7 +193,7 @@ func TestDedupeDropsSecondEventWithSameID(t *testing.T) {
pl := &fkpipeline.Pipeline{Processors: chain} pl := &fkpipeline.Pipeline{Processors: chain}
in := fkevent.Event{ in := fkevent.Event{
ID: "evt-dedupe-1", ID: "evt-dedupe-1",
Kind: fkevent.Kind("observation"), Kind: fkevent.Kind(standards.KindObservation),
Source: "test", Source: "test",
EmittedAt: time.Now().UTC(), EmittedAt: time.Now().UTC(),
Schema: "raw.weatherfeeder.unknown.v1", Schema: "raw.weatherfeeder.unknown.v1",

View File

@@ -5,6 +5,7 @@ import (
"time" "time"
"gitea.maximumdirect.net/ejr/feedkit/event" "gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
) )
func TestFinalizeRoundsWeatherPayloadFloats(t *testing.T) { func TestFinalizeRoundsWeatherPayloadFloats(t *testing.T) {
@@ -14,7 +15,7 @@ func TestFinalizeRoundsWeatherPayloadFloats(t *testing.T) {
in := event.Event{ in := event.Event{
ID: "evt-1", ID: "evt-1",
Kind: event.Kind("observation"), Kind: event.Kind(standards.KindObservation),
Source: "source-a", Source: "source-a",
EmittedAt: time.Date(2026, 3, 28, 12, 0, 0, 0, time.UTC), EmittedAt: time.Date(2026, 3, 28, 12, 0, 0, 0, time.UTC),
Schema: "raw.example.v1", Schema: "raw.example.v1",

View File

@@ -18,7 +18,7 @@ func TestForecastDiscussionNormalizerProducesCanonicalSchema(t *testing.T) {
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{ out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
ID: "evt-discussion-1", ID: "evt-discussion-1",
Kind: event.Kind("forecast_discussion"), Kind: event.Kind(standards.KindForecastDiscussion),
Source: "nws-discussion-test", Source: "nws-discussion-test",
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC), EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
Schema: standards.SchemaRawNWSForecastDiscussionV1, Schema: standards.SchemaRawNWSForecastDiscussionV1,
@@ -33,7 +33,7 @@ func TestForecastDiscussionNormalizerProducesCanonicalSchema(t *testing.T) {
if out.Schema != standards.SchemaWeatherForecastDiscussionV1 { if out.Schema != standards.SchemaWeatherForecastDiscussionV1 {
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherForecastDiscussionV1) t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherForecastDiscussionV1)
} }
if out.Kind != event.Kind("forecast_discussion") { if out.Kind != event.Kind(standards.KindForecastDiscussion) {
t.Fatalf("Kind = %q, want forecast_discussion", out.Kind) t.Fatalf("Kind = %q, want forecast_discussion", out.Kind)
} }
@@ -74,7 +74,7 @@ func TestForecastDiscussionNormalizerProducesCanonicalSchema(t *testing.T) {
func TestForecastDiscussionNormalizerRejectsMissingIssueTime(t *testing.T) { func TestForecastDiscussionNormalizerRejectsMissingIssueTime(t *testing.T) {
_, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{ _, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
ID: "evt-discussion-bad", ID: "evt-discussion-bad",
Kind: event.Kind("forecast_discussion"), Kind: event.Kind(standards.KindForecastDiscussion),
Source: "nws-discussion-test", Source: "nws-discussion-test",
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC), EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
Schema: standards.SchemaRawNWSForecastDiscussionV1, Schema: standards.SchemaRawNWSForecastDiscussionV1,
@@ -93,7 +93,7 @@ func TestForecastDiscussionNormalizerWireShapeHasNoUnexpectedKeys(t *testing.T)
out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{ out, err := (ForecastDiscussionNormalizer{}).Normalize(nil, event.Event{
ID: "evt-discussion-2", ID: "evt-discussion-2",
Kind: event.Kind("forecast_discussion"), Kind: event.Kind(standards.KindForecastDiscussion),
Source: "nws-discussion-test", Source: "nws-discussion-test",
EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC), EmittedAt: time.Date(2026, 3, 28, 19, 25, 0, 0, time.UTC),
Schema: standards.SchemaRawNWSForecastDiscussionV1, Schema: standards.SchemaRawNWSForecastDiscussionV1,

View File

@@ -182,7 +182,7 @@ func TestNormalizeForecastEventBySchemaProducesCanonicalWeatherForecastSchema(t
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
out, err := normalizeForecastEventBySchema(event.Event{ out, err := normalizeForecastEventBySchema(event.Event{
ID: "evt-1", ID: "evt-1",
Kind: event.Kind("forecast"), Kind: event.Kind(standards.KindForecast),
Source: "nws-test", Source: "nws-test",
EmittedAt: time.Date(2026, 3, 16, 18, 0, 0, 0, time.UTC), EmittedAt: time.Date(2026, 3, 16, 18, 0, 0, 0, time.UTC),
Schema: tt.schema, Schema: tt.schema,

View File

@@ -22,7 +22,7 @@ func TestWeatherStoriesNormalizerProducesCanonicalSchemaAndMapsSample(t *testing
if out.Schema != standards.SchemaWeatherStoryV1 { if out.Schema != standards.SchemaWeatherStoryV1 {
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherStoryV1) t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherStoryV1)
} }
if out.Kind != event.Kind("weather_story") { if out.Kind != event.Kind(standards.KindWeatherStory) {
t.Fatalf("Kind = %q, want weather_story", out.Kind) t.Fatalf("Kind = %q, want weather_story", out.Kind)
} }
@@ -118,7 +118,7 @@ func TestWeatherStoriesNormalizerMatch(t *testing.T) {
func weatherStoriesRawEvent(payload string) event.Event { func weatherStoriesRawEvent(payload string) event.Event {
return event.Event{ return event.Event{
ID: "evt-weatherstories-1", ID: "evt-weatherstories-1",
Kind: event.Kind("weather_story"), Kind: event.Kind(standards.KindWeatherStory),
Source: "nws-weatherstories-test", Source: "nws-weatherstories-test",
EmittedAt: time.Date(2026, 5, 30, 9, 5, 0, 0, time.UTC), EmittedAt: time.Date(2026, 5, 30, 9, 5, 0, 0, time.UTC),
Schema: standards.SchemaRawNWSWeatherStoriesV1, Schema: standards.SchemaRawNWSWeatherStoriesV1,

View File

@@ -22,7 +22,6 @@ const (
providerSPC = "spc" providerSPC = "spc"
productConvective = "convective" productConvective = "convective"
outlookNormalizer = "spc convective outlook" outlookNormalizer = "spc convective outlook"
outlookKind = "outlook"
outlookTypeUnknown = 99 outlookTypeUnknown = 99
) )

View File

@@ -32,7 +32,7 @@ func TestConvectiveOutlookNormalizerProducesCanonicalSchemaAndMapsSample(t *test
if out.Schema != standards.SchemaWeatherOutlookV1 { if out.Schema != standards.SchemaWeatherOutlookV1 {
t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherOutlookV1) t.Fatalf("Schema = %q, want %q", out.Schema, standards.SchemaWeatherOutlookV1)
} }
if out.Kind != event.Kind("outlook") { if out.Kind != event.Kind(standards.KindOutlook) {
t.Fatalf("Kind = %q, want outlook", out.Kind) t.Fatalf("Kind = %q, want outlook", out.Kind)
} }
@@ -276,7 +276,7 @@ func spcRawEvent(t *testing.T, bundle spcprovider.RawConvectiveOutlookBundle) ev
effectiveAt := time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC) effectiveAt := time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC)
return event.Event{ return event.Event{
ID: "evt-spc-outlook-1", ID: "evt-spc-outlook-1",
Kind: event.Kind("outlook"), Kind: event.Kind(standards.KindOutlook),
Source: "spc-test", Source: "spc-test",
EmittedAt: time.Date(2026, 6, 11, 20, 5, 0, 0, time.UTC), EmittedAt: time.Date(2026, 6, 11, 20, 5, 0, 0, time.UTC),
EffectiveAt: &effectiveAt, EffectiveAt: &effectiveAt,

View File

@@ -0,0 +1,17 @@
package spc
import (
"os"
"path/filepath"
"testing"
)
func readTestFile(t *testing.T, name string) []byte {
t.Helper()
path := filepath.Join("testdata", name)
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read %s: %v", path, err)
}
return raw
}

View File

@@ -1,8 +1,6 @@
package spc package spc
import ( import (
"os"
"path/filepath"
"strings" "strings"
"testing" "testing"
"time" "time"
@@ -78,13 +76,3 @@ func TestParseISOTimestampTrimsAndReturnsUTC(t *testing.T) {
t.Fatalf("ParseISOTimestamp() = %s, want %s", got, want) t.Fatalf("ParseISOTimestamp() = %s, want %s", got, want)
} }
} }
func readTestFile(t *testing.T, name string) []byte {
t.Helper()
path := filepath.Join("testdata", name)
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read %s: %v", path, err)
}
return raw
}

View File

@@ -48,13 +48,7 @@ func mapObservationEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
writes = append(writes, fksinks.PostgresWrite{ writes = append(writes, fksinks.PostgresWrite{
Table: tableObservations, Table: tableObservations,
Values: map[string]any{ Values: parentEventValues(e, map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
"station_id": nullableString(obs.StationID), "station_id": nullableString(obs.StationID),
"station_name": nullableString(obs.StationName), "station_name": nullableString(obs.StationName),
"observed_at": observedAt, "observed_at": observedAt,
@@ -70,7 +64,7 @@ func mapObservationEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
"visibility_meters": nullableFloat64(obs.VisibilityMeters), "visibility_meters": nullableFloat64(obs.VisibilityMeters),
"relative_humidity_percent": nullableFloat64(obs.RelativeHumidityPercent), "relative_humidity_percent": nullableFloat64(obs.RelativeHumidityPercent),
"apparent_temperature_c": nullableFloat64(obs.ApparentTemperatureC), "apparent_temperature_c": nullableFloat64(obs.ApparentTemperatureC),
}, }),
}) })
for i, pw := range obs.PresentWeather { for i, pw := range obs.PresentWeather {
@@ -109,23 +103,17 @@ func mapForecastEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
writes = append(writes, fksinks.PostgresWrite{ writes = append(writes, fksinks.PostgresWrite{
Table: tableForecasts, Table: tableForecasts,
Values: map[string]any{ Values: parentEventValues(e, map[string]any{
"event_id": e.ID, "location_id": nullableString(run.LocationID),
"event_kind": string(e.Kind), "location_name": nullableString(run.LocationName),
"event_source": e.Source, "issued_at": issuedAt,
"event_schema": e.Schema, "updated_at": nullableTime(run.UpdatedAt),
"event_emitted_at": e.EmittedAt.UTC(), "product": string(run.Product),
"event_effective_at": nullableTime(e.EffectiveAt), "latitude": nullableFloat64(run.Latitude),
"location_id": nullableString(run.LocationID), "longitude": nullableFloat64(run.Longitude),
"location_name": nullableString(run.LocationName), "elevation_meters": nullableFloat64(run.ElevationMeters),
"issued_at": issuedAt, "period_count": len(run.Periods),
"updated_at": nullableTime(run.UpdatedAt), }),
"product": string(run.Product),
"latitude": nullableFloat64(run.Latitude),
"longitude": nullableFloat64(run.Longitude),
"elevation_meters": nullableFloat64(run.ElevationMeters),
"period_count": len(run.Periods),
},
}) })
for i, p := range run.Periods { for i, p := range run.Periods {
@@ -186,13 +174,7 @@ func mapForecastDiscussionEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.KeyMessages)) writes := make([]fksinks.PostgresWrite, 0, 1+len(run.KeyMessages))
writes = append(writes, fksinks.PostgresWrite{ writes = append(writes, fksinks.PostgresWrite{
Table: tableForecastDiscussions, Table: tableForecastDiscussions,
Values: map[string]any{ Values: parentEventValues(e, map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
"office_id": nullableString(run.OfficeID), "office_id": nullableString(run.OfficeID),
"office_name": nullableString(run.OfficeName), "office_name": nullableString(run.OfficeName),
"issued_at": issuedAt, "issued_at": issuedAt,
@@ -205,7 +187,7 @@ func mapForecastDiscussionEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error
"long_term_issued_at": longTermIssuedAt, "long_term_issued_at": longTermIssuedAt,
"long_term_text": longTermText, "long_term_text": longTermText,
"key_message_count": len(run.KeyMessages), "key_message_count": len(run.KeyMessages),
}, }),
}) })
for i, msg := range run.KeyMessages { for i, msg := range run.KeyMessages {
@@ -236,17 +218,11 @@ func mapWeatherStoryEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.Stories)) writes := make([]fksinks.PostgresWrite, 0, 1+len(run.Stories))
writes = append(writes, fksinks.PostgresWrite{ writes = append(writes, fksinks.PostgresWrite{
Table: tableWeatherStoryRuns, Table: tableWeatherStoryRuns,
Values: map[string]any{ Values: parentEventValues(e, map[string]any{
"event_id": e.ID, "office_id": nullableString(run.OfficeID),
"event_kind": string(e.Kind), "as_of": asOf,
"event_source": e.Source, "story_count": len(run.Stories),
"event_schema": e.Schema, }),
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
"office_id": nullableString(run.OfficeID),
"as_of": asOf,
"story_count": len(run.Stories),
},
}) })
for i, story := range run.Stories { for i, story := range run.Stories {
@@ -290,20 +266,14 @@ func mapAlertEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
writes = append(writes, fksinks.PostgresWrite{ writes = append(writes, fksinks.PostgresWrite{
Table: tableAlertRuns, Table: tableAlertRuns,
Values: map[string]any{ Values: parentEventValues(e, map[string]any{
"event_id": e.ID, "location_id": nullableString(run.LocationID),
"event_kind": string(e.Kind), "location_name": nullableString(run.LocationName),
"event_source": e.Source, "as_of": asOf,
"event_schema": e.Schema, "latitude": nullableFloat64(run.Latitude),
"event_emitted_at": e.EmittedAt.UTC(), "longitude": nullableFloat64(run.Longitude),
"event_effective_at": nullableTime(e.EffectiveAt), "alert_count": len(run.Alerts),
"location_id": nullableString(run.LocationID), }),
"location_name": nullableString(run.LocationName),
"as_of": asOf,
"latitude": nullableFloat64(run.Latitude),
"longitude": nullableFloat64(run.Longitude),
"alert_count": len(run.Alerts),
},
}) })
for i, a := range run.Alerts { for i, a := range run.Alerts {
@@ -372,21 +342,15 @@ func mapOutlookEvent(e fkevent.Event) ([]fksinks.PostgresWrite, error) {
writes := make([]fksinks.PostgresWrite, 0, 1+len(run.Outlooks)) writes := make([]fksinks.PostgresWrite, 0, 1+len(run.Outlooks))
writes = append(writes, fksinks.PostgresWrite{ writes = append(writes, fksinks.PostgresWrite{
Table: tableOutlookRuns, Table: tableOutlookRuns,
Values: map[string]any{ Values: parentEventValues(e, map[string]any{
"event_id": e.ID, "location_id": nullableString(run.LocationID),
"event_kind": string(e.Kind), "location_name": nullableString(run.LocationName),
"event_source": e.Source, "latitude": nullableFloat64(run.Latitude),
"event_schema": e.Schema, "longitude": nullableFloat64(run.Longitude),
"event_emitted_at": e.EmittedAt.UTC(), "as_of": asOf,
"event_effective_at": nullableTime(e.EffectiveAt), "issued_at": nullableTime(run.IssuedAt),
"location_id": nullableString(run.LocationID), "outlook_count": len(run.Outlooks),
"location_name": nullableString(run.LocationName), }),
"latitude": nullableFloat64(run.Latitude),
"longitude": nullableFloat64(run.Longitude),
"as_of": asOf,
"issued_at": nullableTime(run.IssuedAt),
"outlook_count": len(run.Outlooks),
},
}) })
for i, outlook := range run.Outlooks { for i, outlook := range run.Outlooks {
@@ -488,6 +452,21 @@ func decodePayload[T any](payload any) (T, error) {
return out, nil return out, nil
} }
func parentEventValues(e fkevent.Event, values map[string]any) map[string]any {
out := map[string]any{
"event_id": e.ID,
"event_kind": string(e.Kind),
"event_source": e.Source,
"event_schema": e.Schema,
"event_emitted_at": e.EmittedAt.UTC(),
"event_effective_at": nullableTime(e.EffectiveAt),
}
for k, v := range values {
out[k] = v
}
return out
}
func nullableDiscussionSection(section *model.WeatherForecastDiscussionSection) (any, any, any) { func nullableDiscussionSection(section *model.WeatherForecastDiscussionSection) (any, any, any) {
if section == nil { if section == nil {
return nil, nil, nil return nil, nil, nil

View File

@@ -27,7 +27,7 @@ func TestMapPostgresEventObservationStructPayload(t *testing.T) {
PresentWeather: []model.PresentWeather{{Raw: map[string]any{"a": 1, "b": "x"}}}, PresentWeather: []model.PresentWeather{{Raw: map[string]any{"a": 1, "b": "x"}}},
} }
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherObservationV1, "observation", obs)) writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherObservationV1, standards.KindObservation, obs))
if err != nil { if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err) t.Fatalf("mapPostgresEvent() error = %v", err)
} }
@@ -74,7 +74,7 @@ func TestMapPostgresEventForecastStructPayload(t *testing.T) {
}, },
} }
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, "forecast", run)) writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, standards.KindForecast, run))
if err != nil { if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err) t.Fatalf("mapPostgresEvent() error = %v", err)
} }
@@ -122,7 +122,7 @@ func TestMapPostgresEventAlertStructPayload(t *testing.T) {
}, },
} }
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherAlertV1, "alert", run)) writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherAlertV1, standards.KindAlert, run))
if err != nil { if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err) t.Fatalf("mapPostgresEvent() error = %v", err)
} }
@@ -163,7 +163,7 @@ func TestMapPostgresEventForecastDiscussionStructPayload(t *testing.T) {
LongTerm: &model.WeatherForecastDiscussionSection{Text: "Long term text"}, LongTerm: &model.WeatherForecastDiscussionSection{Text: "Long term text"},
} }
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, "forecast_discussion", run)) writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, standards.KindForecastDiscussion, run))
if err != nil { if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err) t.Fatalf("mapPostgresEvent() error = %v", err)
} }
@@ -213,7 +213,7 @@ func TestMapPostgresEventWeatherStoryStructPayload(t *testing.T) {
}, },
} }
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, "weather_story", run)) writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, standards.KindWeatherStory, run))
if err != nil { if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err) t.Fatalf("mapPostgresEvent() error = %v", err)
} }
@@ -290,7 +290,7 @@ func TestMapPostgresEventOutlookStructPayload(t *testing.T) {
}, },
} }
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, "outlook", run)) writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, standards.KindOutlook, run))
if err != nil { if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err) t.Fatalf("mapPostgresEvent() error = %v", err)
} }
@@ -332,7 +332,7 @@ func TestMapPostgresEventOutlookStructPayload(t *testing.T) {
} }
func TestMapPostgresEventOutlookRejectsMissingAsOf(t *testing.T) { func TestMapPostgresEventOutlookRejectsMissingAsOf(t *testing.T) {
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, "outlook", model.WeatherOutlookRun{})) _, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, standards.KindOutlook, model.WeatherOutlookRun{}))
if err == nil { if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want missing asOf error") t.Fatalf("mapPostgresEvent() error = nil, want missing asOf error")
} }
@@ -381,7 +381,7 @@ func TestMapPostgresEventOutlookRejectsMissingIDAndProvider(t *testing.T) {
AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC), AsOf: time.Date(2026, 6, 11, 19, 45, 0, 0, time.UTC),
Outlooks: []model.WeatherOutlook{outlook}, Outlooks: []model.WeatherOutlook{outlook},
} }
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, "outlook", run)) _, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, standards.KindOutlook, run))
if err == nil { if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want %q", tt.wantErr) t.Fatalf("mapPostgresEvent() error = nil, want %q", tt.wantErr)
} }
@@ -405,7 +405,7 @@ func TestMapPostgresEventOutlookRejectsMissingRequiredTimes(t *testing.T) {
Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-91,38],[-90,38],[-90,39],[-91,39],[-91,38]]]}`), Geometry: json.RawMessage(`{"type":"Polygon","coordinates":[[[-91,38],[-90,38],[-90,39],[-91,39],[-91,38]]]}`),
}}, }},
} }
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, "outlook", run)) _, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, standards.KindOutlook, run))
if err == nil { if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want missing time error") t.Fatalf("mapPostgresEvent() error = nil, want missing time error")
} }
@@ -430,7 +430,7 @@ func TestMapPostgresEventOutlookRejectsEmptyGeometry(t *testing.T) {
ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC), ExpiresAt: time.Date(2026, 6, 12, 12, 0, 0, 0, time.UTC),
}}, }},
} }
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, "outlook", run)) _, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherOutlookV1, standards.KindOutlook, run))
if err == nil { if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want geometry error") t.Fatalf("mapPostgresEvent() error = nil, want geometry error")
} }
@@ -440,7 +440,7 @@ func TestMapPostgresEventOutlookRejectsEmptyGeometry(t *testing.T) {
} }
func TestMapPostgresEventWeatherStoryRejectsMissingAsOf(t *testing.T) { func TestMapPostgresEventWeatherStoryRejectsMissingAsOf(t *testing.T) {
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, "weather_story", model.WeatherStoryRun{})) _, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, standards.KindWeatherStory, model.WeatherStoryRun{}))
if err == nil { if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want missing asOf error") t.Fatalf("mapPostgresEvent() error = nil, want missing asOf error")
} }
@@ -454,7 +454,7 @@ func TestMapPostgresEventWeatherStoryRejectsMissingStoryTimes(t *testing.T) {
AsOf: time.Date(2026, 5, 30, 9, 0, 34, 0, time.UTC), AsOf: time.Date(2026, 5, 30, 9, 0, 34, 0, time.UTC),
Stories: []model.WeatherStory{{Title: "missing times"}}, Stories: []model.WeatherStory{{Title: "missing times"}},
} }
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, "weather_story", run)) _, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherStoryV1, standards.KindWeatherStory, run))
if err == nil { if err == nil {
t.Fatalf("mapPostgresEvent() error = nil, want missing story times error") t.Fatalf("mapPostgresEvent() error = nil, want missing story times error")
} }
@@ -484,7 +484,7 @@ func TestMapPostgresEventMapPayload(t *testing.T) {
t.Fatalf("json.Unmarshal() error = %v", err) t.Fatalf("json.Unmarshal() error = %v", err)
} }
writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, "forecast", payload)) writes, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, standards.KindForecast, payload))
if err != nil { if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err) t.Fatalf("mapPostgresEvent() error = %v", err)
} }
@@ -499,7 +499,7 @@ func TestMapPostgresEventMapPayload(t *testing.T) {
} }
func TestMapPostgresEventUnknownSchemaNoOp(t *testing.T) { func TestMapPostgresEventUnknownSchemaNoOp(t *testing.T) {
writes, err := mapPostgresEvent(context.Background(), testEvent("weather.unknown.v1", "observation", map[string]any{"x": 1})) writes, err := mapPostgresEvent(context.Background(), testEvent("weather.unknown.v1", standards.KindObservation, map[string]any{"x": 1}))
if err != nil { if err != nil {
t.Fatalf("mapPostgresEvent() error = %v", err) t.Fatalf("mapPostgresEvent() error = %v", err)
} }
@@ -509,7 +509,7 @@ func TestMapPostgresEventUnknownSchemaNoOp(t *testing.T) {
} }
func TestMapPostgresEventMalformedPayload(t *testing.T) { func TestMapPostgresEventMalformedPayload(t *testing.T) {
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, "forecast", "bad")) _, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastV1, standards.KindForecast, "bad"))
if err == nil { if err == nil {
t.Fatalf("mapPostgresEvent() expected error for malformed payload") t.Fatalf("mapPostgresEvent() expected error for malformed payload")
} }
@@ -519,7 +519,7 @@ func TestMapPostgresEventMalformedPayload(t *testing.T) {
} }
func TestMapPostgresEventForecastDiscussionMalformedPayload(t *testing.T) { func TestMapPostgresEventForecastDiscussionMalformedPayload(t *testing.T) {
_, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, "forecast_discussion", "bad")) _, err := mapPostgresEvent(context.Background(), testEvent(standards.SchemaWeatherForecastDiscussionV1, standards.KindForecastDiscussion, "bad"))
if err == nil { if err == nil {
t.Fatalf("mapPostgresEvent() expected error for malformed payload") t.Fatalf("mapPostgresEvent() expected error for malformed payload")
} }

View File

@@ -16,20 +16,20 @@ type pollDriverRegistration struct {
} }
var pollDriverRegistrations = []pollDriverRegistration{ var pollDriverRegistrations = []pollDriverRegistration{
{driver: "nws_observation", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewObservationSource(cfg) }}, {driver: nws.DriverObservation, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewObservationSource(cfg) }},
{driver: "nws_alerts", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewAlertsSource(cfg) }}, {driver: nws.DriverAlerts, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewAlertsSource(cfg) }},
{driver: "nws_forecast_hourly", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewHourlyForecastSource(cfg) }}, {driver: nws.DriverForecastHourly, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewHourlyForecastSource(cfg) }},
{driver: "nws_forecast_narrative", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewNarrativeForecastSource(cfg) }}, {driver: nws.DriverForecastNarrative, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewNarrativeForecastSource(cfg) }},
{driver: "nws_forecast_discussion", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { {driver: nws.DriverForecastDiscussion, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
return nws.NewForecastDiscussionSource(cfg) return nws.NewForecastDiscussionSource(cfg)
}}, }},
{driver: "nws_weatherstories", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewWeatherStoriesSource(cfg) }}, {driver: nws.DriverWeatherStories, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return nws.NewWeatherStoriesSource(cfg) }},
{driver: "openmeteo_observation", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewObservationSource(cfg) }}, {driver: openmeteo.DriverObservation, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewObservationSource(cfg) }},
{driver: "openmeteo_forecast", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewForecastSource(cfg) }}, {driver: openmeteo.DriverForecast, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { return openmeteo.NewForecastSource(cfg) }},
{driver: "openweather_observation", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { {driver: openweather.DriverObservation, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
return openweather.NewObservationSource(cfg) return openweather.NewObservationSource(cfg)
}}, }},
{driver: "spc_convective_outlook", factory: func(cfg config.SourceConfig) (fksource.PollSource, error) { {driver: spc.DriverConvectiveOutlook, factory: func(cfg config.SourceConfig) (fksource.PollSource, error) {
return spc.NewConvectiveOutlookSource(cfg) return spc.NewConvectiveOutlookSource(cfg)
}}, }},
} }

View File

@@ -6,57 +6,29 @@ import (
"gitea.maximumdirect.net/ejr/feedkit/config" "gitea.maximumdirect.net/ejr/feedkit/config"
fksource "gitea.maximumdirect.net/ejr/feedkit/sources" fksource "gitea.maximumdirect.net/ejr/feedkit/sources"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/openweather"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/spc"
) )
func TestRegisterBuiltinsRegistersNWSHourlyForecastDriver(t *testing.T) { func TestRegisterBuiltinsRegistersCurrentPollDrivers(t *testing.T) {
reg := fksource.NewRegistry() reg := fksource.NewRegistry()
RegisterBuiltins(reg) RegisterBuiltins(reg)
in, err := reg.BuildInput(sourceConfigForDriver("nws_forecast_hourly")) if len(pollDriverRegistrations) == 0 {
if err != nil { t.Fatalf("pollDriverRegistrations is empty")
t.Fatalf("BuildInput(nws_forecast_hourly) error = %v", err)
} }
if _, ok := in.(fksource.PollSource); !ok {
t.Fatalf("BuildInput(nws_forecast_hourly) type = %T, want PollSource", in)
}
}
func TestRegisterBuiltinsRegistersNWSNarrativeForecastDriver(t *testing.T) { for _, tt := range pollDriverRegistrations {
reg := fksource.NewRegistry() tt := tt
RegisterBuiltins(reg) t.Run(tt.driver, func(t *testing.T) {
in, err := reg.BuildInput(sourceConfigForDriver(tt.driver))
in, err := reg.BuildInput(sourceConfigForDriver("nws_forecast_narrative")) if err != nil {
if err != nil { t.Fatalf("BuildInput(%q) error = %v", tt.driver, err)
t.Fatalf("BuildInput(nws_forecast_narrative) error = %v", err) }
} if _, ok := in.(fksource.PollSource); !ok {
if _, ok := in.(fksource.PollSource); !ok { t.Fatalf("BuildInput(%q) type = %T, want PollSource", tt.driver, in)
t.Fatalf("BuildInput(nws_forecast_narrative) type = %T, want PollSource", in) }
} })
}
func TestRegisterBuiltinsRegistersNWSForecastDiscussionDriver(t *testing.T) {
reg := fksource.NewRegistry()
RegisterBuiltins(reg)
in, err := reg.BuildInput(sourceConfigForDriver("nws_forecast_discussion"))
if err != nil {
t.Fatalf("BuildInput(nws_forecast_discussion) error = %v", err)
}
if _, ok := in.(fksource.PollSource); !ok {
t.Fatalf("BuildInput(nws_forecast_discussion) type = %T, want PollSource", in)
}
}
func TestRegisterBuiltinsRegistersNWSWeatherStoriesDriver(t *testing.T) {
reg := fksource.NewRegistry()
RegisterBuiltins(reg)
in, err := reg.BuildInput(sourceConfigForDriver("nws_weatherstories"))
if err != nil {
t.Fatalf("BuildInput(nws_weatherstories) error = %v", err)
}
if _, ok := in.(fksource.PollSource); !ok {
t.Fatalf("BuildInput(nws_weatherstories) type = %T, want PollSource", in)
} }
} }
@@ -73,44 +45,16 @@ func TestRegisterBuiltinsDoesNotRegisterLegacyNWSForecastDriver(t *testing.T) {
} }
} }
func TestRegisterBuiltinsRegistersAllCurrentDrivers(t *testing.T) {
reg := fksource.NewRegistry()
RegisterBuiltins(reg)
drivers := []string{
"nws_observation",
"nws_alerts",
"nws_forecast_hourly",
"nws_forecast_narrative",
"nws_forecast_discussion",
"nws_weatherstories",
"openmeteo_observation",
"openmeteo_forecast",
"openweather_observation",
"spc_convective_outlook",
}
for _, driver := range drivers {
in, err := reg.BuildInput(sourceConfigForDriver(driver))
if err != nil {
t.Fatalf("BuildInput(%s) error = %v", driver, err)
}
if _, ok := in.(fksource.PollSource); !ok {
t.Fatalf("BuildInput(%s) type = %T, want PollSource", driver, in)
}
}
}
func sourceConfigForDriver(driver string) config.SourceConfig { func sourceConfigForDriver(driver string) config.SourceConfig {
url := "https://example.invalid" url := "https://example.invalid"
if driver == "openweather_observation" { if driver == openweather.DriverObservation {
url = "https://example.invalid?units=metric" url = "https://example.invalid?units=metric"
} }
params := map[string]any{ params := map[string]any{
"url": url, "url": url,
"user_agent": "test-agent", "user_agent": "test-agent",
} }
if driver == "spc_convective_outlook" { if driver == spc.DriverConvectiveOutlook {
params["latitude"] = 38.6239 params["latitude"] = 38.6239
params["longitude"] = -90.3571 params["longitude"] = -90.3571
} }

View File

@@ -0,0 +1,32 @@
package sources
import (
"os"
"strings"
"testing"
)
func TestDocumentedRegisteredSourceDrivers(t *testing.T) {
docs := map[string]string{
"docs/config.md": readDoc(t, "../../docs/config.md"),
"docs/internal/sources.md": readDoc(t, "../../docs/internal/sources.md"),
}
for _, reg := range pollDriverRegistrations {
for path, doc := range docs {
if !strings.Contains(doc, reg.driver) {
t.Fatalf("%s missing source driver %q", path, reg.driver)
}
}
}
}
func readDoc(t *testing.T, path string) string {
t.Helper()
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("ReadFile(%s) error = %v", path, err)
}
return string(raw)
}

View File

@@ -0,0 +1,58 @@
package httpconfig
import (
"fmt"
"strings"
"time"
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/transport"
)
// Settings contains common HTTP client config for sources that fetch multiple documents.
type Settings struct {
Name string
UserAgent string
Timeout time.Duration
BodyLimitBytes int64
}
func Parse(driver string, cfg config.SourceConfig) (Settings, error) {
name := strings.TrimSpace(cfg.Name)
if name == "" {
return Settings{}, fmt.Errorf("%s: name is required", driver)
}
if cfg.Params == nil {
return Settings{}, fmt.Errorf("%s %q: params are required", driver, name)
}
userAgent, ok := cfg.ParamString("user_agent", "userAgent")
if !ok {
return Settings{}, fmt.Errorf("%s %q: params.user_agent is required", driver, name)
}
timeout := transport.DefaultHTTPTimeout
if _, exists := cfg.Params["http_timeout"]; exists {
var ok bool
timeout, ok = cfg.ParamDuration("http_timeout")
if !ok || timeout <= 0 {
return Settings{}, fmt.Errorf("source %q: params.http_timeout must be a positive duration", name)
}
}
bodyLimit := transport.DefaultHTTPResponseBodyLimitBytes
if _, exists := cfg.Params["http_response_body_limit_bytes"]; exists {
rawLimit, ok := cfg.ParamInt("http_response_body_limit_bytes")
if !ok || rawLimit <= 0 {
return Settings{}, fmt.Errorf("source %q: params.http_response_body_limit_bytes must be a positive integer", name)
}
bodyLimit = int64(rawLimit)
}
return Settings{
Name: name,
UserAgent: userAgent,
Timeout: timeout,
BodyLimitBytes: bodyLimit,
}, nil
}

View File

@@ -0,0 +1,115 @@
package httpconfig
import (
"strings"
"testing"
"time"
"gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/transport"
)
func TestParseUsesRequiredValuesAndDefaults(t *testing.T) {
got, err := Parse("test_driver", config.SourceConfig{
Name: " test-source ",
Params: map[string]any{
"user_agent": "test-agent",
},
})
if err != nil {
t.Fatalf("Parse() error = %v", err)
}
if got.Name != "test-source" {
t.Fatalf("Name = %q, want test-source", got.Name)
}
if got.UserAgent != "test-agent" {
t.Fatalf("UserAgent = %q, want test-agent", got.UserAgent)
}
if got.Timeout != transport.DefaultHTTPTimeout {
t.Fatalf("Timeout = %s, want %s", got.Timeout, transport.DefaultHTTPTimeout)
}
if got.BodyLimitBytes != transport.DefaultHTTPResponseBodyLimitBytes {
t.Fatalf("BodyLimitBytes = %d, want %d", got.BodyLimitBytes, transport.DefaultHTTPResponseBodyLimitBytes)
}
}
func TestParseUsesAliasesAndOverrides(t *testing.T) {
got, err := Parse("test_driver", config.SourceConfig{
Name: "test-source",
Params: map[string]any{
"userAgent": "test-agent",
"http_timeout": "2s",
"http_response_body_limit_bytes": 2048,
},
})
if err != nil {
t.Fatalf("Parse() error = %v", err)
}
if got.UserAgent != "test-agent" {
t.Fatalf("UserAgent = %q, want test-agent", got.UserAgent)
}
if got.Timeout != 2*time.Second {
t.Fatalf("Timeout = %s, want 2s", got.Timeout)
}
if got.BodyLimitBytes != 2048 {
t.Fatalf("BodyLimitBytes = %d, want 2048", got.BodyLimitBytes)
}
}
func TestParseRejectsInvalidConfig(t *testing.T) {
tests := []struct {
name string
cfg config.SourceConfig
wantErr string
}{
{
name: "missing name",
cfg: config.SourceConfig{Params: map[string]any{"user_agent": "test-agent"}},
wantErr: "test_driver: name is required",
},
{
name: "missing params",
cfg: config.SourceConfig{Name: "test-source"},
wantErr: `test_driver "test-source": params are required`,
},
{
name: "missing user agent",
cfg: config.SourceConfig{Name: "test-source", Params: map[string]any{}},
wantErr: `test_driver "test-source": params.user_agent is required`,
},
{
name: "invalid timeout",
cfg: config.SourceConfig{
Name: "test-source",
Params: map[string]any{
"user_agent": "test-agent",
"http_timeout": "0s",
},
},
wantErr: `source "test-source": params.http_timeout must be a positive duration`,
},
{
name: "invalid body limit",
cfg: config.SourceConfig{
Name: "test-source",
Params: map[string]any{
"user_agent": "test-agent",
"http_response_body_limit_bytes": 0,
},
},
wantErr: `source "test-source": params.http_response_body_limit_bytes must be a positive integer`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := Parse("test_driver", tt.cfg)
if err == nil {
t.Fatalf("Parse() error = nil, want %q", tt.wantErr)
}
if !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("Parse() error = %q, want %q", err, tt.wantErr)
}
})
}
}

View File

@@ -26,10 +26,8 @@ type AlertsSource struct {
} }
func NewAlertsSource(cfg config.SourceConfig) (*AlertsSource, error) { func NewAlertsSource(cfg config.SourceConfig) (*AlertsSource, error) {
const driver = "nws_alerts"
// NWS alerts responses are GeoJSON-ish; allow fallback to plain JSON as well. // NWS alerts responses are GeoJSON-ish; allow fallback to plain JSON as well.
hs, err := fksources.NewHTTPSource(driver, cfg, "application/geo+json, application/json") hs, err := fksources.NewHTTPSource(DriverAlerts, cfg, "application/geo+json, application/json")
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -40,7 +38,7 @@ func NewAlertsSource(cfg config.SourceConfig) (*AlertsSource, error) {
func (s *AlertsSource) Name() string { return s.http.Name } func (s *AlertsSource) Name() string { return s.http.Name }
// Kinds is used for routing/policy. // Kinds is used for routing/policy.
func (s *AlertsSource) Kinds() []event.Kind { return []event.Kind{event.Kind("alert")} } func (s *AlertsSource) Kinds() []event.Kind { return []event.Kind{event.Kind(standards.KindAlert)} }
func (s *AlertsSource) Poll(ctx context.Context) ([]event.Event, error) { func (s *AlertsSource) Poll(ctx context.Context) ([]event.Event, error) {
raw, meta, changed, err := s.fetchRaw(ctx) raw, meta, changed, err := s.fetchRaw(ctx)
@@ -71,7 +69,7 @@ func (s *AlertsSource) Poll(ctx context.Context) ([]event.Event, error) {
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt) eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent( return fksources.SingleEvent(
event.Kind("alert"), event.Kind(standards.KindAlert),
s.http.Name, s.http.Name,
standards.SchemaRawNWSAlertsV1, standards.SchemaRawNWSAlertsV1,
eventID, eventID,

View File

@@ -0,0 +1,11 @@
package nws
// Source driver strings registered by weatherfeeder for NWS sources.
const (
DriverObservation = "nws_observation"
DriverAlerts = "nws_alerts"
DriverForecastHourly = "nws_forecast_hourly"
DriverForecastNarrative = "nws_forecast_narrative"
DriverForecastDiscussion = "nws_forecast_discussion"
DriverWeatherStories = "nws_weatherstories"
)

View File

@@ -10,6 +10,7 @@ import (
"gitea.maximumdirect.net/ejr/feedkit/event" "gitea.maximumdirect.net/ejr/feedkit/event"
fksources "gitea.maximumdirect.net/ejr/feedkit/sources" fksources "gitea.maximumdirect.net/ejr/feedkit/sources"
nwscommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/nws" nwscommon "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/nws"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
) )
const nwsForecastAccept = "application/geo+json, application/json" const nwsForecastAccept = "application/geo+json, application/json"
@@ -44,7 +45,9 @@ func newForecastSource(cfg config.SourceConfig, driver, rawSchema string) (*fore
func (s *forecastSource) Name() string { return s.http.Name } func (s *forecastSource) Name() string { return s.http.Name }
func (s *forecastSource) Kinds() []event.Kind { return []event.Kind{event.Kind("forecast")} } func (s *forecastSource) Kinds() []event.Kind {
return []event.Kind{event.Kind(standards.KindForecast)}
}
func (s *forecastSource) Poll(ctx context.Context) ([]event.Event, error) { func (s *forecastSource) Poll(ctx context.Context) ([]event.Event, error) {
raw, meta, changed, err := s.fetchRaw(ctx) raw, meta, changed, err := s.fetchRaw(ctx)
@@ -69,7 +72,7 @@ func (s *forecastSource) Poll(ctx context.Context) ([]event.Event, error) {
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt) eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent( return fksources.SingleEvent(
event.Kind("forecast"), event.Kind(standards.KindForecast),
s.http.Name, s.http.Name,
s.rawSchema, s.rawSchema,
eventID, eventID,

View File

@@ -20,9 +20,7 @@ type ForecastDiscussionSource struct {
} }
func NewForecastDiscussionSource(cfg config.SourceConfig) (*ForecastDiscussionSource, error) { func NewForecastDiscussionSource(cfg config.SourceConfig) (*ForecastDiscussionSource, error) {
const driver = "nws_forecast_discussion" hs, err := fksources.NewHTTPSource(DriverForecastDiscussion, cfg, "text/html, application/xhtml+xml")
hs, err := fksources.NewHTTPSource(driver, cfg, "text/html, application/xhtml+xml")
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -33,7 +31,7 @@ func NewForecastDiscussionSource(cfg config.SourceConfig) (*ForecastDiscussionSo
func (s *ForecastDiscussionSource) Name() string { return s.http.Name } func (s *ForecastDiscussionSource) Name() string { return s.http.Name }
func (s *ForecastDiscussionSource) Kinds() []event.Kind { func (s *ForecastDiscussionSource) Kinds() []event.Kind {
return []event.Kind{event.Kind("forecast_discussion")} return []event.Kind{event.Kind(standards.KindForecastDiscussion)}
} }
func (s *ForecastDiscussionSource) Poll(ctx context.Context) ([]event.Event, error) { func (s *ForecastDiscussionSource) Poll(ctx context.Context) ([]event.Event, error) {
@@ -57,7 +55,7 @@ func (s *ForecastDiscussionSource) Poll(ctx context.Context) ([]event.Event, err
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt) eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent( return fksources.SingleEvent(
event.Kind("forecast_discussion"), event.Kind(standards.KindForecastDiscussion),
s.http.Name, s.http.Name,
standards.SchemaRawNWSForecastDiscussionV1, standards.SchemaRawNWSForecastDiscussionV1,
eventID, eventID,

View File

@@ -27,7 +27,7 @@ func TestForecastDiscussionSourcePollEmitsExpectedEvent(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("NewForecastDiscussionSource() error = %v", err) t.Fatalf("NewForecastDiscussionSource() error = %v", err)
} }
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("forecast_discussion") { if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindForecastDiscussion) {
t.Fatalf("Kinds() = %#v, want [forecast_discussion]", got) t.Fatalf("Kinds() = %#v, want [forecast_discussion]", got)
} }
@@ -40,7 +40,7 @@ func TestForecastDiscussionSourcePollEmitsExpectedEvent(t *testing.T) {
} }
got := events[0] got := events[0]
if got.Kind != event.Kind("forecast_discussion") { if got.Kind != event.Kind(standards.KindForecastDiscussion) {
t.Fatalf("Kind = %q, want forecast_discussion", got.Kind) t.Fatalf("Kind = %q, want forecast_discussion", got.Kind)
} }
if got.Schema != standards.SchemaRawNWSForecastDiscussionV1 { if got.Schema != standards.SchemaRawNWSForecastDiscussionV1 {
@@ -117,7 +117,7 @@ func TestForecastDiscussionSourcePollRejectsInvalidHTML(t *testing.T) {
func forecastDiscussionSourceConfig(url string) config.SourceConfig { func forecastDiscussionSourceConfig(url string) config.SourceConfig {
return config.SourceConfig{ return config.SourceConfig{
Name: "test-forecast-discussion-source", Name: "test-forecast-discussion-source",
Driver: "nws_forecast_discussion", Driver: DriverForecastDiscussion,
Mode: config.SourceModePoll, Mode: config.SourceModePoll,
Params: map[string]any{ Params: map[string]any{
"url": url, "url": url,

View File

@@ -18,8 +18,7 @@ type HourlyForecastSource struct {
} }
func NewHourlyForecastSource(cfg config.SourceConfig) (*HourlyForecastSource, error) { func NewHourlyForecastSource(cfg config.SourceConfig) (*HourlyForecastSource, error) {
const driver = "nws_forecast_hourly" src, err := newForecastSource(cfg, DriverForecastHourly, standards.SchemaRawNWSHourlyForecastV1)
src, err := newForecastSource(cfg, driver, standards.SchemaRawNWSHourlyForecastV1)
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@@ -18,8 +18,7 @@ type NarrativeForecastSource struct {
} }
func NewNarrativeForecastSource(cfg config.SourceConfig) (*NarrativeForecastSource, error) { func NewNarrativeForecastSource(cfg config.SourceConfig) (*NarrativeForecastSource, error) {
const driver = "nws_forecast_narrative" src, err := newForecastSource(cfg, DriverForecastNarrative, standards.SchemaRawNWSNarrativeForecastV1)
src, err := newForecastSource(cfg, driver, standards.SchemaRawNWSNarrativeForecastV1)
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@@ -26,7 +26,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
}{ }{
{ {
name: "hourly", name: "hourly",
driver: "nws_forecast_hourly", driver: DriverForecastHourly,
wantSchema: standards.SchemaRawNWSHourlyForecastV1, wantSchema: standards.SchemaRawNWSHourlyForecastV1,
newSource: func(cfg config.SourceConfig) (forecastPoller, error) { newSource: func(cfg config.SourceConfig) (forecastPoller, error) {
return NewHourlyForecastSource(cfg) return NewHourlyForecastSource(cfg)
@@ -34,7 +34,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
}, },
{ {
name: "narrative", name: "narrative",
driver: "nws_forecast_narrative", driver: DriverForecastNarrative,
wantSchema: standards.SchemaRawNWSNarrativeForecastV1, wantSchema: standards.SchemaRawNWSNarrativeForecastV1,
newSource: func(cfg config.SourceConfig) (forecastPoller, error) { newSource: func(cfg config.SourceConfig) (forecastPoller, error) {
return NewNarrativeForecastSource(cfg) return NewNarrativeForecastSource(cfg)
@@ -55,7 +55,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
} }
if ks, ok := src.(interface{ Kinds() []event.Kind }); !ok { if ks, ok := src.(interface{ Kinds() []event.Kind }); !ok {
t.Fatalf("source does not implement Kinds()") t.Fatalf("source does not implement Kinds()")
} else if gotKinds := ks.Kinds(); len(gotKinds) != 1 || gotKinds[0] != event.Kind("forecast") { } else if gotKinds := ks.Kinds(); len(gotKinds) != 1 || gotKinds[0] != event.Kind(standards.KindForecast) {
t.Fatalf("Kinds() = %#v, want [forecast]", gotKinds) t.Fatalf("Kinds() = %#v, want [forecast]", gotKinds)
} }
@@ -69,7 +69,7 @@ func TestForecastSourcesEmitExpectedSchemaAndPreferGeneratedAt(t *testing.T) {
if got[0].Schema != tt.wantSchema { if got[0].Schema != tt.wantSchema {
t.Fatalf("Poll() schema = %q, want %q", got[0].Schema, tt.wantSchema) t.Fatalf("Poll() schema = %q, want %q", got[0].Schema, tt.wantSchema)
} }
if got[0].Kind != event.Kind("forecast") { if got[0].Kind != event.Kind(standards.KindForecast) {
t.Fatalf("Poll() kind = %q, want forecast", got[0].Kind) t.Fatalf("Poll() kind = %q, want forecast", got[0].Kind)
} }
@@ -117,7 +117,7 @@ func TestForecastSourcePollEffectiveAtFallbackOrder(t *testing.T) {
})) }))
defer srv.Close() defer srv.Close()
src, err := NewHourlyForecastSource(forecastSourceConfig("nws_forecast_hourly", srv.URL)) src, err := NewHourlyForecastSource(forecastSourceConfig(DriverForecastHourly, srv.URL))
if err != nil { if err != nil {
t.Fatalf("NewHourlyForecastSource() error = %v", err) t.Fatalf("NewHourlyForecastSource() error = %v", err)
} }
@@ -148,7 +148,7 @@ func TestForecastSourcePollMetadataDecodeFailureStillEmitsRawEvent(t *testing.T)
})) }))
defer srv.Close() defer srv.Close()
src, err := NewNarrativeForecastSource(forecastSourceConfig("nws_forecast_narrative", srv.URL)) src, err := NewNarrativeForecastSource(forecastSourceConfig(DriverForecastNarrative, srv.URL))
if err != nil { if err != nil {
t.Fatalf("NewNarrativeForecastSource() error = %v", err) t.Fatalf("NewNarrativeForecastSource() error = %v", err)
} }

View File

@@ -20,9 +20,7 @@ type ObservationSource struct {
} }
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) { func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
const driver = "nws_observation" hs, err := fksources.NewHTTPSource(DriverObservation, cfg, "application/geo+json, application/json")
hs, err := fksources.NewHTTPSource(driver, cfg, "application/geo+json, application/json")
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -32,7 +30,9 @@ func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
func (s *ObservationSource) Name() string { return s.http.Name } func (s *ObservationSource) Name() string { return s.http.Name }
func (s *ObservationSource) Kinds() []event.Kind { return []event.Kind{event.Kind("observation")} } func (s *ObservationSource) Kinds() []event.Kind {
return []event.Kind{event.Kind(standards.KindObservation)}
}
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) { func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
raw, meta, changed, err := s.fetchRaw(ctx) raw, meta, changed, err := s.fetchRaw(ctx)
@@ -54,7 +54,7 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
eventID := fksources.DefaultEventID(meta.ID, s.http.Name, effectiveAt, emittedAt) eventID := fksources.DefaultEventID(meta.ID, s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent( return fksources.SingleEvent(
event.Kind("observation"), event.Kind(standards.KindObservation),
s.http.Name, s.http.Name,
standards.SchemaRawNWSObservationV1, standards.SchemaRawNWSObservationV1,
eventID, eventID,

View File

@@ -8,6 +8,7 @@ import (
"gitea.maximumdirect.net/ejr/feedkit/config" "gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event" "gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
) )
func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) { func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
@@ -31,7 +32,7 @@ func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
src, err := NewObservationSource(config.SourceConfig{ src, err := NewObservationSource(config.SourceConfig{
Name: "NWSObservationTest", Name: "NWSObservationTest",
Driver: "nws_observation", Driver: DriverObservation,
Mode: config.SourceModePoll, Mode: config.SourceModePoll,
Params: map[string]any{ Params: map[string]any{
"url": srv.URL, "url": srv.URL,
@@ -41,7 +42,7 @@ func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("NewObservationSource() error = %v", err) t.Fatalf("NewObservationSource() error = %v", err)
} }
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("observation") { if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindObservation) {
t.Fatalf("Kinds() = %#v, want [observation]", got) t.Fatalf("Kinds() = %#v, want [observation]", got)
} }
@@ -52,7 +53,7 @@ func TestObservationSourcePollReturnsNoEventsOn304(t *testing.T) {
if len(first) != 1 { if len(first) != 1 {
t.Fatalf("first Poll() len = %d, want 1", len(first)) t.Fatalf("first Poll() len = %d, want 1", len(first))
} }
if first[0].Kind != event.Kind("observation") { if first[0].Kind != event.Kind(standards.KindObservation) {
t.Fatalf("first Poll() kind = %q", first[0].Kind) t.Fatalf("first Poll() kind = %q", first[0].Kind)
} }

View File

@@ -22,9 +22,7 @@ type WeatherStoriesSource struct {
} }
func NewWeatherStoriesSource(cfg config.SourceConfig) (*WeatherStoriesSource, error) { func NewWeatherStoriesSource(cfg config.SourceConfig) (*WeatherStoriesSource, error) {
const driver = "nws_weatherstories" hs, err := fksources.NewHTTPSource(DriverWeatherStories, cfg, "application/geo+json, application/json")
hs, err := fksources.NewHTTPSource(driver, cfg, "application/geo+json, application/json")
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -35,7 +33,7 @@ func NewWeatherStoriesSource(cfg config.SourceConfig) (*WeatherStoriesSource, er
func (s *WeatherStoriesSource) Name() string { return s.http.Name } func (s *WeatherStoriesSource) Name() string { return s.http.Name }
func (s *WeatherStoriesSource) Kinds() []event.Kind { func (s *WeatherStoriesSource) Kinds() []event.Kind {
return []event.Kind{event.Kind("weather_story")} return []event.Kind{event.Kind(standards.KindWeatherStory)}
} }
func (s *WeatherStoriesSource) Poll(ctx context.Context) ([]event.Event, error) { func (s *WeatherStoriesSource) Poll(ctx context.Context) ([]event.Event, error) {
@@ -61,7 +59,7 @@ func (s *WeatherStoriesSource) Poll(ctx context.Context) ([]event.Event, error)
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt) eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent( return fksources.SingleEvent(
event.Kind("weather_story"), event.Kind(standards.KindWeatherStory),
s.http.Name, s.http.Name,
standards.SchemaRawNWSWeatherStoriesV1, standards.SchemaRawNWSWeatherStoriesV1,
eventID, eventID,

View File

@@ -28,7 +28,7 @@ func TestWeatherStoriesSourcePollEmitsExpectedEventAndPrefersLatestUpdateTime(t
if err != nil { if err != nil {
t.Fatalf("NewWeatherStoriesSource() error = %v", err) t.Fatalf("NewWeatherStoriesSource() error = %v", err)
} }
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("weather_story") { if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindWeatherStory) {
t.Fatalf("Kinds() = %#v, want [weather_story]", got) t.Fatalf("Kinds() = %#v, want [weather_story]", got)
} }
@@ -41,7 +41,7 @@ func TestWeatherStoriesSourcePollEmitsExpectedEventAndPrefersLatestUpdateTime(t
} }
got := events[0] got := events[0]
if got.Kind != event.Kind("weather_story") { if got.Kind != event.Kind(standards.KindWeatherStory) {
t.Fatalf("Kind = %q, want weather_story", got.Kind) t.Fatalf("Kind = %q, want weather_story", got.Kind)
} }
if got.Schema != standards.SchemaRawNWSWeatherStoriesV1 { if got.Schema != standards.SchemaRawNWSWeatherStoriesV1 {
@@ -148,7 +148,7 @@ func TestWeatherStoriesSourcePollMetadataDecodeFailureStillEmitsRawEvent(t *test
func weatherStoriesSourceConfig(url string) config.SourceConfig { func weatherStoriesSourceConfig(url string) config.SourceConfig {
return config.SourceConfig{ return config.SourceConfig{
Name: "test-weatherstories-source", Name: "test-weatherstories-source",
Driver: "nws_weatherstories", Driver: DriverWeatherStories,
Mode: config.SourceModePoll, Mode: config.SourceModePoll,
Params: map[string]any{ Params: map[string]any{
"url": url, "url": url,

View File

@@ -0,0 +1,7 @@
package openmeteo
// Source driver strings registered by weatherfeeder for Open-Meteo sources.
const (
DriverObservation = "openmeteo_observation"
DriverForecast = "openmeteo_forecast"
)

View File

@@ -19,9 +19,7 @@ type ForecastSource struct {
} }
func NewForecastSource(cfg config.SourceConfig) (*ForecastSource, error) { func NewForecastSource(cfg config.SourceConfig) (*ForecastSource, error) {
const driver = "openmeteo_forecast" hs, err := fksources.NewHTTPSource(DriverForecast, cfg, "application/json")
hs, err := fksources.NewHTTPSource(driver, cfg, "application/json")
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -31,7 +29,9 @@ func NewForecastSource(cfg config.SourceConfig) (*ForecastSource, error) {
func (s *ForecastSource) Name() string { return s.http.Name } func (s *ForecastSource) Name() string { return s.http.Name }
func (s *ForecastSource) Kinds() []event.Kind { return []event.Kind{event.Kind("forecast")} } func (s *ForecastSource) Kinds() []event.Kind {
return []event.Kind{event.Kind(standards.KindForecast)}
}
func (s *ForecastSource) Poll(ctx context.Context) ([]event.Event, error) { func (s *ForecastSource) Poll(ctx context.Context) ([]event.Event, error) {
raw, meta, changed, err := s.fetchRaw(ctx) raw, meta, changed, err := s.fetchRaw(ctx)
@@ -55,7 +55,7 @@ func (s *ForecastSource) Poll(ctx context.Context) ([]event.Event, error) {
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt) eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent( return fksources.SingleEvent(
event.Kind("forecast"), event.Kind(standards.KindForecast),
s.http.Name, s.http.Name,
standards.SchemaRawOpenMeteoHourlyForecastV1, standards.SchemaRawOpenMeteoHourlyForecastV1,
eventID, eventID,

View File

@@ -19,9 +19,7 @@ type ObservationSource struct {
} }
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) { func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
const driver = "openmeteo_observation" hs, err := fksources.NewHTTPSource(DriverObservation, cfg, "application/json")
hs, err := fksources.NewHTTPSource(driver, cfg, "application/json")
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -31,7 +29,9 @@ func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
func (s *ObservationSource) Name() string { return s.http.Name } func (s *ObservationSource) Name() string { return s.http.Name }
func (s *ObservationSource) Kinds() []event.Kind { return []event.Kind{event.Kind("observation")} } func (s *ObservationSource) Kinds() []event.Kind {
return []event.Kind{event.Kind(standards.KindObservation)}
}
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) { func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
raw, meta, changed, err := s.fetchRaw(ctx) raw, meta, changed, err := s.fetchRaw(ctx)
@@ -52,7 +52,7 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt) eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent( return fksources.SingleEvent(
event.Kind("observation"), event.Kind(standards.KindObservation),
s.http.Name, s.http.Name,
standards.SchemaRawOpenMeteoCurrentV1, standards.SchemaRawOpenMeteoCurrentV1,
eventID, eventID,

View File

@@ -5,12 +5,13 @@ import (
"gitea.maximumdirect.net/ejr/feedkit/config" "gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event" "gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
) )
func TestObservationSourceAdvertisesKinds(t *testing.T) { func TestObservationSourceAdvertisesKinds(t *testing.T) {
src, err := NewObservationSource(config.SourceConfig{ src, err := NewObservationSource(config.SourceConfig{
Name: "openmeteo-observation-test", Name: "openmeteo-observation-test",
Driver: "openmeteo_observation", Driver: DriverObservation,
Mode: config.SourceModePoll, Mode: config.SourceModePoll,
Params: map[string]any{ Params: map[string]any{
"url": "https://example.invalid", "url": "https://example.invalid",
@@ -20,7 +21,7 @@ func TestObservationSourceAdvertisesKinds(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("NewObservationSource() error = %v", err) t.Fatalf("NewObservationSource() error = %v", err)
} }
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("observation") { if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindObservation) {
t.Fatalf("Kinds() = %#v, want [observation]", got) t.Fatalf("Kinds() = %#v, want [observation]", got)
} }
} }
@@ -28,7 +29,7 @@ func TestObservationSourceAdvertisesKinds(t *testing.T) {
func TestForecastSourceAdvertisesKinds(t *testing.T) { func TestForecastSourceAdvertisesKinds(t *testing.T) {
src, err := NewForecastSource(config.SourceConfig{ src, err := NewForecastSource(config.SourceConfig{
Name: "openmeteo-forecast-test", Name: "openmeteo-forecast-test",
Driver: "openmeteo_forecast", Driver: DriverForecast,
Mode: config.SourceModePoll, Mode: config.SourceModePoll,
Params: map[string]any{ Params: map[string]any{
"url": "https://example.invalid", "url": "https://example.invalid",
@@ -38,7 +39,7 @@ func TestForecastSourceAdvertisesKinds(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("NewForecastSource() error = %v", err) t.Fatalf("NewForecastSource() error = %v", err)
} }
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("forecast") { if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindForecast) {
t.Fatalf("Kinds() = %#v, want [forecast]", got) t.Fatalf("Kinds() = %#v, want [forecast]", got)
} }
} }

View File

@@ -0,0 +1,6 @@
package openweather
// Source driver strings registered by weatherfeeder for OpenWeather sources.
const (
DriverObservation = "openweather_observation"
)

View File

@@ -19,9 +19,7 @@ type ObservationSource struct {
} }
func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) { func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
const driver = "openweather_observation" hs, err := fksources.NewHTTPSource(DriverObservation, cfg, "application/json")
hs, err := fksources.NewHTTPSource(driver, cfg, "application/json")
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -35,7 +33,9 @@ func NewObservationSource(cfg config.SourceConfig) (*ObservationSource, error) {
func (s *ObservationSource) Name() string { return s.http.Name } func (s *ObservationSource) Name() string { return s.http.Name }
func (s *ObservationSource) Kinds() []event.Kind { return []event.Kind{event.Kind("observation")} } func (s *ObservationSource) Kinds() []event.Kind {
return []event.Kind{event.Kind(standards.KindObservation)}
}
func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) { func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
if err := owcommon.RequireMetricUnits(s.http.URL); err != nil { if err := owcommon.RequireMetricUnits(s.http.URL); err != nil {
@@ -60,7 +60,7 @@ func (s *ObservationSource) Poll(ctx context.Context) ([]event.Event, error) {
eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt) eventID := fksources.DefaultEventID("", s.http.Name, effectiveAt, emittedAt)
return fksources.SingleEvent( return fksources.SingleEvent(
event.Kind("observation"), event.Kind(standards.KindObservation),
s.http.Name, s.http.Name,
standards.SchemaRawOpenWeatherCurrentV1, standards.SchemaRawOpenWeatherCurrentV1,
eventID, eventID,

View File

@@ -5,12 +5,13 @@ import (
"gitea.maximumdirect.net/ejr/feedkit/config" "gitea.maximumdirect.net/ejr/feedkit/config"
"gitea.maximumdirect.net/ejr/feedkit/event" "gitea.maximumdirect.net/ejr/feedkit/event"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards"
) )
func TestObservationSourceAdvertisesKinds(t *testing.T) { func TestObservationSourceAdvertisesKinds(t *testing.T) {
src, err := NewObservationSource(config.SourceConfig{ src, err := NewObservationSource(config.SourceConfig{
Name: "openweather-observation-test", Name: "openweather-observation-test",
Driver: "openweather_observation", Driver: DriverObservation,
Mode: config.SourceModePoll, Mode: config.SourceModePoll,
Params: map[string]any{ Params: map[string]any{
"url": "https://example.invalid?units=metric", "url": "https://example.invalid?units=metric",
@@ -20,7 +21,7 @@ func TestObservationSourceAdvertisesKinds(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("NewObservationSource() error = %v", err) t.Fatalf("NewObservationSource() error = %v", err)
} }
if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind("observation") { if got := src.Kinds(); len(got) != 1 || got[0] != event.Kind(standards.KindObservation) {
t.Fatalf("Kinds() = %#v, want [observation]", got) t.Fatalf("Kinds() = %#v, want [observation]", got)
} }
} }

View File

@@ -16,12 +16,11 @@ import (
fksources "gitea.maximumdirect.net/ejr/feedkit/sources" fksources "gitea.maximumdirect.net/ejr/feedkit/sources"
"gitea.maximumdirect.net/ejr/feedkit/transport" "gitea.maximumdirect.net/ejr/feedkit/transport"
spcprovider "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/spc" spcprovider "gitea.maximumdirect.net/ejr/weatherfeeder/internal/providers/spc"
"gitea.maximumdirect.net/ejr/weatherfeeder/internal/sources/internal/httpconfig"
"gitea.maximumdirect.net/ejr/weatherfeeder/standards" "gitea.maximumdirect.net/ejr/weatherfeeder/standards"
) )
const ( const (
driverConvectiveOutlook = "spc_convective_outlook"
acceptGeoJSON = "application/geo+json, application/json" acceptGeoJSON = "application/geo+json, application/json"
acceptDiscussion = "text/html, application/xhtml+xml" acceptDiscussion = "text/html, application/xhtml+xml"
acceptRSS = "application/rss+xml, application/xml, text/xml" acceptRSS = "application/rss+xml, application/xml, text/xml"
@@ -56,53 +55,27 @@ type ConvectiveOutlookSource struct {
} }
func NewConvectiveOutlookSource(cfg config.SourceConfig) (*ConvectiveOutlookSource, error) { func NewConvectiveOutlookSource(cfg config.SourceConfig) (*ConvectiveOutlookSource, error) {
name := strings.TrimSpace(cfg.Name) httpSettings, err := httpconfig.Parse(DriverConvectiveOutlook, cfg)
if name == "" { if err != nil {
return nil, fmt.Errorf("%s: name is required", driverConvectiveOutlook) return nil, err
}
if cfg.Params == nil {
return nil, fmt.Errorf("%s %q: params are required", driverConvectiveOutlook, name)
}
userAgent, ok := cfg.ParamString("user_agent", "userAgent")
if !ok {
return nil, fmt.Errorf("%s %q: params.user_agent is required", driverConvectiveOutlook, name)
} }
latitude, err := requireFloatParam(cfg, "latitude") latitude, err := requireFloatParam(cfg, "latitude")
if err != nil { if err != nil {
return nil, fmt.Errorf("%s %q: %w", driverConvectiveOutlook, name, err) return nil, fmt.Errorf("%s %q: %w", DriverConvectiveOutlook, httpSettings.Name, err)
} }
longitude, err := requireFloatParam(cfg, "longitude") longitude, err := requireFloatParam(cfg, "longitude")
if err != nil { if err != nil {
return nil, fmt.Errorf("%s %q: %w", driverConvectiveOutlook, name, err) return nil, fmt.Errorf("%s %q: %w", DriverConvectiveOutlook, httpSettings.Name, err)
}
timeout := transport.DefaultHTTPTimeout
if _, exists := cfg.Params["http_timeout"]; exists {
var ok bool
timeout, ok = cfg.ParamDuration("http_timeout")
if !ok || timeout <= 0 {
return nil, fmt.Errorf("source %q: params.http_timeout must be a positive duration", name)
}
}
bodyLimit := transport.DefaultHTTPResponseBodyLimitBytes
if _, exists := cfg.Params["http_response_body_limit_bytes"]; exists {
rawLimit, ok := cfg.ParamInt("http_response_body_limit_bytes")
if !ok || rawLimit <= 0 {
return nil, fmt.Errorf("source %q: params.http_response_body_limit_bytes must be a positive integer", name)
}
bodyLimit = int64(rawLimit)
} }
geoJSONProducts, err := configuredGeoJSONProducts(cfg) geoJSONProducts, err := configuredGeoJSONProducts(cfg)
if err != nil { if err != nil {
return nil, fmt.Errorf("%s %q: %w", driverConvectiveOutlook, name, err) return nil, fmt.Errorf("%s %q: %w", DriverConvectiveOutlook, httpSettings.Name, err)
} }
discussions, err := configuredDiscussionProducts(cfg) discussions, err := configuredDiscussionProducts(cfg)
if err != nil { if err != nil {
return nil, fmt.Errorf("%s %q: %w", driverConvectiveOutlook, name, err) return nil, fmt.Errorf("%s %q: %w", DriverConvectiveOutlook, httpSettings.Name, err)
} }
rssURL := "" rssURL := ""
@@ -114,14 +87,14 @@ func NewConvectiveOutlookSource(cfg config.SourceConfig) (*ConvectiveOutlookSour
locationName, _ := cfg.ParamString("location_name", "locationName") locationName, _ := cfg.ParamString("location_name", "locationName")
return &ConvectiveOutlookSource{ return &ConvectiveOutlookSource{
name: name, name: httpSettings.Name,
userAgent: userAgent, userAgent: httpSettings.UserAgent,
locationID: locationID, locationID: locationID,
locationName: locationName, locationName: locationName,
latitude: latitude, latitude: latitude,
longitude: longitude, longitude: longitude,
client: transport.NewHTTPClient(timeout), client: transport.NewHTTPClient(httpSettings.Timeout),
bodyLimit: bodyLimit, bodyLimit: httpSettings.BodyLimitBytes,
geoJSONProducts: geoJSONProducts, geoJSONProducts: geoJSONProducts,
discussions: discussions, discussions: discussions,
rssURL: rssURL, rssURL: rssURL,
@@ -131,7 +104,7 @@ func NewConvectiveOutlookSource(cfg config.SourceConfig) (*ConvectiveOutlookSour
func (s *ConvectiveOutlookSource) Name() string { return s.name } func (s *ConvectiveOutlookSource) Name() string { return s.name }
func (s *ConvectiveOutlookSource) Kinds() []event.Kind { func (s *ConvectiveOutlookSource) Kinds() []event.Kind {
return []event.Kind{event.Kind("outlook")} return []event.Kind{event.Kind(standards.KindOutlook)}
} }
func (s *ConvectiveOutlookSource) Poll(ctx context.Context) ([]event.Event, error) { func (s *ConvectiveOutlookSource) Poll(ctx context.Context) ([]event.Event, error) {
@@ -221,7 +194,7 @@ func (s *ConvectiveOutlookSource) Poll(ctx context.Context) ([]event.Event, erro
eventID := fksources.DefaultEventID("", s.name, &effectiveAt, emittedAt) eventID := fksources.DefaultEventID("", s.name, &effectiveAt, emittedAt)
return fksources.SingleEvent( return fksources.SingleEvent(
event.Kind("outlook"), event.Kind(standards.KindOutlook),
s.name, s.name,
standards.SchemaRawSPCConvectiveOutlookV1, standards.SchemaRawSPCConvectiveOutlookV1,
eventID, eventID,

View File

@@ -22,7 +22,7 @@ func TestConvectiveOutlookSourceKinds(t *testing.T) {
t.Fatalf("NewConvectiveOutlookSource() error = %v", err) t.Fatalf("NewConvectiveOutlookSource() error = %v", err)
} }
got := src.Kinds() got := src.Kinds()
if len(got) != 1 || got[0] != event.Kind("outlook") { if len(got) != 1 || got[0] != event.Kind(standards.KindOutlook) {
t.Fatalf("Kinds() = %#v, want [outlook]", got) t.Fatalf("Kinds() = %#v, want [outlook]", got)
} }
} }
@@ -57,7 +57,7 @@ func TestConvectiveOutlookSourcePollEmitsRawBundle(t *testing.T) {
t.Fatalf("Poll() returned %d events, want 1", len(events)) t.Fatalf("Poll() returned %d events, want 1", len(events))
} }
got := events[0] got := events[0]
if got.Kind != event.Kind("outlook") { if got.Kind != event.Kind(standards.KindOutlook) {
t.Fatalf("Kind = %q, want outlook", got.Kind) t.Fatalf("Kind = %q, want outlook", got.Kind)
} }
if got.Schema != standards.SchemaRawSPCConvectiveOutlookV1 { if got.Schema != standards.SchemaRawSPCConvectiveOutlookV1 {
@@ -277,7 +277,7 @@ func convectiveOutlookConfig(extra map[string]any) config.SourceConfig {
} }
return config.SourceConfig{ return config.SourceConfig{
Name: "spc-test", Name: "spc-test",
Driver: driverConvectiveOutlook, Driver: DriverConvectiveOutlook,
Mode: config.SourceModePoll, Mode: config.SourceModePoll,
Params: params, Params: params,
} }

View File

@@ -0,0 +1,6 @@
package spc
// Source driver strings registered by weatherfeeder for SPC sources.
const (
DriverConvectiveOutlook = "spc_convective_outlook"
)

69
standards/docs_test.go Normal file
View File

@@ -0,0 +1,69 @@
package standards
import (
"go/ast"
"go/parser"
"go/token"
"os"
"strconv"
"strings"
"testing"
)
func TestDocumentedEventSchemas(t *testing.T) {
raw, err := os.ReadFile("../docs/integrations/events.md")
if err != nil {
t.Fatalf("ReadFile(events.md) error = %v", err)
}
doc := string(raw)
schemas := schemaConstants(t)
for _, schema := range schemas {
if !strings.Contains(doc, schema) {
t.Fatalf("docs/integrations/events.md missing schema %q", schema)
}
}
}
func schemaConstants(t *testing.T) []string {
t.Helper()
file, err := parser.ParseFile(token.NewFileSet(), "schema.go", nil, 0)
if err != nil {
t.Fatalf("ParseFile(schema.go) error = %v", err)
}
var out []string
ast.Inspect(file, func(n ast.Node) bool {
valueSpec, ok := n.(*ast.ValueSpec)
if !ok {
return true
}
for i, name := range valueSpec.Names {
if !strings.HasPrefix(name.Name, "Schema") || schemaConstantNotInCurrentContract(name.Name) {
continue
}
if i >= len(valueSpec.Values) {
t.Fatalf("schema constant %s has no explicit value", name.Name)
}
lit, ok := valueSpec.Values[i].(*ast.BasicLit)
if !ok || lit.Kind != token.STRING {
t.Fatalf("schema constant %s is not a string literal", name.Name)
}
schema, err := strconv.Unquote(lit.Value)
if err != nil {
t.Fatalf("schema constant %s value is not a quoted string: %v", name.Name, err)
}
out = append(out, schema)
}
return true
})
if len(out) == 0 {
t.Fatalf("no schema constants found")
}
return out
}
func schemaConstantNotInCurrentContract(name string) bool {
return name == "SchemaRawOpenWeatherHourlyForecastV1"
}

11
standards/kind.go Normal file
View File

@@ -0,0 +1,11 @@
package standards
// Event kind strings used by weatherfeeder events and routing policy.
const (
KindObservation = "observation"
KindForecast = "forecast"
KindForecastDiscussion = "forecast_discussion"
KindWeatherStory = "weather_story"
KindAlert = "alert"
KindOutlook = "outlook"
)