148 lines
3.8 KiB
Go
148 lines
3.8 KiB
Go
package sources
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
|
|
"gitea.maximumdirect.net/ejr/feedkit/config"
|
|
"gitea.maximumdirect.net/ejr/feedkit/transport"
|
|
)
|
|
|
|
// HTTPSource is a reusable helper for polling HTTP-backed sources.
|
|
//
|
|
// It centralizes generic source config parsing (`params.url`,
|
|
// `params.user_agent`, and optional `params.conditional`), default HTTP client
|
|
// setup, and conditional GET validator handling. Concrete daemon sources remain
|
|
// responsible for decoding the response body and constructing events.
|
|
type HTTPSource struct {
|
|
Driver string
|
|
Name string
|
|
URL string
|
|
UserAgent string
|
|
Accept string
|
|
Conditional bool
|
|
Client *http.Client
|
|
|
|
mu sync.Mutex
|
|
validators transport.HTTPValidators
|
|
}
|
|
|
|
// NewHTTPSource builds a generic HTTP polling helper from SourceConfig.
|
|
//
|
|
// Required params:
|
|
// - params.url
|
|
// - params.user_agent
|
|
//
|
|
// Optional params:
|
|
// - params.conditional (default true): enable conditional GET using cached
|
|
// ETag / Last-Modified validators
|
|
func NewHTTPSource(driver string, cfg config.SourceConfig, accept string) (*HTTPSource, error) {
|
|
name := strings.TrimSpace(cfg.Name)
|
|
if name == "" {
|
|
return nil, fmt.Errorf("%s: name is required", driver)
|
|
}
|
|
if cfg.Params == nil {
|
|
return nil, fmt.Errorf("%s %q: params are required (need params.url and params.user_agent)", driver, cfg.Name)
|
|
}
|
|
|
|
url, ok := cfg.ParamString("url", "URL")
|
|
if !ok {
|
|
return nil, fmt.Errorf("%s %q: params.url is required", driver, cfg.Name)
|
|
}
|
|
|
|
userAgent, ok := cfg.ParamString("user_agent", "userAgent")
|
|
if !ok {
|
|
return nil, fmt.Errorf("%s %q: params.user_agent is required", driver, cfg.Name)
|
|
}
|
|
|
|
conditional, err := parseConditionalParam(cfg)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return &HTTPSource{
|
|
Driver: driver,
|
|
Name: name,
|
|
URL: url,
|
|
UserAgent: userAgent,
|
|
Accept: accept,
|
|
Conditional: conditional,
|
|
Client: transport.NewHTTPClient(transport.DefaultHTTPTimeout),
|
|
}, nil
|
|
}
|
|
|
|
// FetchBytesIfChanged fetches the configured URL and reports whether the
|
|
// upstream content changed. An unchanged 304 response returns changed=false
|
|
// with no body and no error.
|
|
func (s *HTTPSource) FetchBytesIfChanged(ctx context.Context) ([]byte, bool, error) {
|
|
client := s.Client
|
|
if client == nil {
|
|
client = transport.NewHTTPClient(transport.DefaultHTTPTimeout)
|
|
}
|
|
|
|
s.mu.Lock()
|
|
validators := s.validators
|
|
s.mu.Unlock()
|
|
|
|
body, changed, next, err := transport.FetchBodyIfChanged(
|
|
ctx,
|
|
client,
|
|
s.URL,
|
|
s.UserAgent,
|
|
s.Accept,
|
|
s.Conditional,
|
|
validators,
|
|
)
|
|
if err != nil {
|
|
return nil, false, fmt.Errorf("%s %q: %w", s.Driver, s.Name, err)
|
|
}
|
|
|
|
if s.Conditional {
|
|
s.mu.Lock()
|
|
s.validators = next
|
|
s.mu.Unlock()
|
|
}
|
|
|
|
return body, changed, nil
|
|
}
|
|
|
|
// FetchJSONIfChanged fetches the configured URL and returns the raw response
|
|
// body as json.RawMessage when content changed. An unchanged 304 response
|
|
// returns changed=false with a nil body and no error.
|
|
func (s *HTTPSource) FetchJSONIfChanged(ctx context.Context) (json.RawMessage, bool, error) {
|
|
body, changed, err := s.FetchBytesIfChanged(ctx)
|
|
if err != nil || !changed {
|
|
return nil, changed, err
|
|
}
|
|
return json.RawMessage(body), true, nil
|
|
}
|
|
|
|
func parseConditionalParam(cfg config.SourceConfig) (bool, error) {
|
|
raw, ok := cfg.Params["conditional"]
|
|
if !ok || raw == nil {
|
|
return true, nil
|
|
}
|
|
|
|
switch v := raw.(type) {
|
|
case bool:
|
|
return v, nil
|
|
case string:
|
|
s := strings.TrimSpace(v)
|
|
if s == "" {
|
|
return false, fmt.Errorf("source %q: params.conditional must be a boolean", cfg.Name)
|
|
}
|
|
parsed, err := strconv.ParseBool(s)
|
|
if err != nil {
|
|
return false, fmt.Errorf("source %q: params.conditional must be a boolean", cfg.Name)
|
|
}
|
|
return parsed, nil
|
|
default:
|
|
return false, fmt.Errorf("source %q: params.conditional must be a boolean", cfg.Name)
|
|
}
|
|
}
|