Added support for Postgres polling sources
This commit is contained in:
@@ -4,18 +4,15 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/ejr/feedkit/config"
|
||||
"gitea.maximumdirect.net/ejr/feedkit/event"
|
||||
_ "github.com/lib/pq"
|
||||
pgconn "gitea.maximumdirect.net/ejr/feedkit/internal/postgres"
|
||||
)
|
||||
|
||||
const postgresInitTimeout = 5 * time.Second
|
||||
|
||||
type postgresTx interface {
|
||||
ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
|
||||
Commit() error
|
||||
@@ -73,8 +70,8 @@ func (w *sqlTxWrapper) Rollback() error {
|
||||
return w.tx.Rollback()
|
||||
}
|
||||
|
||||
var openPostgresDB = func(dsn string) (postgresDB, error) {
|
||||
db, err := sql.Open("postgres", dsn)
|
||||
var openPostgresDB = func(ctx context.Context, cfg pgconn.ConnConfig) (postgresDB, error) {
|
||||
db, err := pgconn.Open(ctx, cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -111,12 +108,11 @@ func NewPostgresSinkFromConfig(cfg config.SinkConfig, schemaDef PostgresSchema)
|
||||
return nil, fmt.Errorf("postgres sink %q: compile schema: %w", cfg.Name, err)
|
||||
}
|
||||
|
||||
dsn, err := buildPostgresDSN(uri, username, password)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("postgres sink %q: build dsn: %w", cfg.Name, err)
|
||||
}
|
||||
|
||||
db, err := openPostgresDB(dsn)
|
||||
db, err := openPostgresDB(context.Background(), pgconn.ConnConfig{
|
||||
URI: uri,
|
||||
Username: username,
|
||||
Password: password,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("postgres sink %q: open db: %w", cfg.Name, err)
|
||||
}
|
||||
@@ -264,13 +260,9 @@ func (p *PostgresSink) PruneAllOlderThan(ctx context.Context, cutoff time.Time)
|
||||
}
|
||||
|
||||
func (p *PostgresSink) initialize() error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), postgresInitTimeout)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if err := p.db.PingContext(ctx); err != nil {
|
||||
return fmt.Errorf("postgres sink %q: ping db: %w", p.name, err)
|
||||
}
|
||||
|
||||
for _, tableName := range p.schema.tableOrder {
|
||||
tbl := p.schema.tables[tableName]
|
||||
|
||||
@@ -302,21 +294,6 @@ func (p *PostgresSink) lookupTable(table string) (postgresTableCompiled, error)
|
||||
return tbl, nil
|
||||
}
|
||||
|
||||
func buildPostgresDSN(uri, username, password string) (string, error) {
|
||||
u, err := url.Parse(strings.TrimSpace(uri))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid uri: %w", err)
|
||||
}
|
||||
if u.Scheme == "" {
|
||||
return "", fmt.Errorf("invalid uri: missing scheme")
|
||||
}
|
||||
if u.Host == "" {
|
||||
return "", fmt.Errorf("invalid uri: missing host")
|
||||
}
|
||||
u.User = url.UserPassword(username, password)
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
func parsePostgresPruneWindow(cfg config.SinkConfig) (time.Duration, error) {
|
||||
raw, ok := cfg.Params["prune"]
|
||||
if !ok || raw == nil {
|
||||
|
||||
Reference in New Issue
Block a user