Add distributor upload adapter

This commit is contained in:
2026-06-07 23:34:21 +00:00
parent a2ba6f5382
commit 64cae8c4d9
5 changed files with 525 additions and 1 deletions

View File

@@ -72,8 +72,10 @@ Run `gofmt -w` on changed Go files before committing.
Prefer the Go standard library. Add dependencies only when they materially
improve correctness, interoperability, security, or maintainability.
Current external dependency:
Current external dependencies:
- `gitea.maximumdirect.net/eric/distributor` for distributor source bundle
construction and HTTP upload client behavior.
- `gopkg.in/yaml.v3` for YAML configuration parsing.
When adding a dependency:

2
go.mod
View File

@@ -3,3 +3,5 @@ module gitea.maximumdirect.net/eric/weatherreporter
go 1.26
require gopkg.in/yaml.v3 v3.0.1
require gitea.maximumdirect.net/eric/distributor v0.4.0

48
go.sum
View File

@@ -1,3 +1,51 @@
gitea.maximumdirect.net/eric/distributor v0.4.0 h1:SRrTFjVLMv4wFZmMLwnaYtJaQlQ/wsB4OzcaSEEb524=
gitea.maximumdirect.net/eric/distributor v0.4.0/go.mod h1:G03FCFZPHpsUKC6SeMgTdbfNRpPQBdyTtDUj04e1Tu8=
github.com/aws/aws-sdk-go-v2 v1.41.9 h1:/rYeyO2+HrMztAmxAq9++XJtFMqSIpSsNA0yDGALYq4=
github.com/aws/aws-sdk-go-v2 v1.41.9/go.mod h1:+HsoOEX80qAVUitj1A2DhCNTjmb3edVyuDypb6LNEeo=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 h1:h5+3VT69KUBK24grGuuA5saDJTj2IIjLb9au668Fo5I=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11/go.mod h1:dnakxebH6UwFvcvujL0LVggYQ8nEvBGjU4G/V79Nv94=
github.com/aws/aws-sdk-go-v2/config v1.32.20 h1:8VMDnWc/kEzxsI/1ngGM9mG81a8IGmIHD8KLcYGwagc=
github.com/aws/aws-sdk-go-v2/config v1.32.20/go.mod h1:PuwEpciweIXGULWeOeSTXtSbH4CW9mWdWrhdCKQI1sM=
github.com/aws/aws-sdk-go-v2/credentials v1.19.19 h1:yuFzSV1U0aRNYCQGVaTY2zW2M/L93pYHnXnrJUphYhU=
github.com/aws/aws-sdk-go-v2/credentials v1.19.19/go.mod h1:7y63L1kGzeoDlJaQ3Z578KrnmfBut96JjvJUzGwR+YE=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.25 h1:0w6dCiO8iez+YKwRhRBlL1CH/E3GTfdkuzrwj1by8vo=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.25/go.mod h1:9FDWUothyr5RCRAHc45XOiVCzUR8n/IhCYX+uVqw6vk=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.25 h1:Uii3frf9ztec/ABM2/FSH9/z7PLzxfpG8h4RpkUFflQ=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.25/go.mod h1:G6kntsA2GorAxDPbap6xgB2F+amSLUF8GJTi7PUoX44=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.25 h1:r1+/l6m+WaUJF9HISEsNOLHSNj5EXYQxK8VX6Cz9NlA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.25/go.mod h1:cKf+D+NMDK1LndD7BowHbBZPgR9V0/5HubH0PFWvA+c=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.26 h1:A1PmWU2zfkIm9EyFlJncFXL4W4phML+h8KjltUsCvNQ=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.26/go.mod h1:dY4MRzXEizrD4hqtpKvWVGPX7QleSGGVY+EBolo1RmM=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.10 h1:d5/908OJ4bXg8lyjeMPvXetEKqoDoLi5Owy1zNue3yg=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.10/go.mod h1:a57l7Hwh+FWI+we50g5NPJHYUKeJKfXbc4w8SyXu8Ig=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 h1:W/EyPFl9A5rXrtoilfwHYEvzHER+K4SpBPtMXi24Mos=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18/go.mod h1:UG50K+pvd/uy6xExbobg0rjqFBFZe6I3l75EPDZw4tg=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.25 h1:dD3dhHNglpd98gs72my22Ndqi1hqQGllFFg1F+twfxg=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.25/go.mod h1:0yAbjPfd64gG7mj85RW+fMEYdfBgCRZw8g/oWcL1pjc=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 h1:2pQEbwf+/6EDbiit/GcBE2K4IUpMZymaA0kOz3xK978=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25/go.mod h1:KvT6NCcQ0EZ+ZkVRrlBMt04Po3ok23YELEp7WimhLhM=
github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2 h1:ie4ElCmUKS26pzrZcIk/lmt4yWjAqLLcawstyQCh298=
github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2/go.mod h1:zjsomFeX5duj+4PlMB+o4JoWTIx+G0XMyzjYrUbQkN0=
github.com/aws/aws-sdk-go-v2/service/signin v1.1.1 h1:1VwbP3qMNfxUDEXWki4rCE5iA+44VA1lokTz9HasGzw=
github.com/aws/aws-sdk-go-v2/service/signin v1.1.1/go.mod h1:vUtyoSj0OPji3kjIVSc/GlKuWEiL33f/WFxl6dmpy/A=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.19 h1:N6pIsdFOW1Kd9S4KyFKXdGRBojPPxkP32+uHFWLv4Hc=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.19/go.mod h1:3gt5WJArFooNmyLONS+h/R4J+o86II8du38IgCwj9dE=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.2 h1:hc+lBYiiTr8Zk4MTzIsQ92MeDWCIDvWGmzKUWOaBcOg=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.2/go.mod h1:hU6fqB3OJA6/ePheD47LQnxvjYk6br6PtQxs+Q9ojvk=
github.com/aws/aws-sdk-go-v2/service/sts v1.42.3 h1:ErklX/7uhSbkAAeyQD/Y1OoQ9hO3SJXQNEgksORW3Js=
github.com/aws/aws-sdk-go-v2/service/sts v1.42.3/go.mod h1:ULe4HCzfKPiR6R3HEurE3b1upEkuk8AkMrOKtaOxKO8=
github.com/aws/smithy-go v1.26.0 h1:9ouqbi+NyKP7fV3Te7UElCwdAb6Y8uk7LGwPE5tVe/s=
github.com/aws/smithy-go v1.26.0/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU=
github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA=
github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE=
github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

View File

@@ -0,0 +1,229 @@
// Package distributor adapts weatherreporter report artifacts to distributor uploads.
package distributor
import (
"context"
"errors"
"fmt"
"net/http"
"os"
"strings"
"time"
distributorbundle "gitea.maximumdirect.net/eric/distributor/pkg/bundle"
distributorupload "gitea.maximumdirect.net/eric/distributor/pkg/upload"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
)
type Client struct {
Endpoint string
TokenEnv string
Timeout time.Duration
newUploadClient uploadClientFactory
}
type UploadRequest struct {
BundleID string
IdempotencyKey string
SourcePath string
BundlePath string
}
type UploadResult struct {
RunID string
Status string
}
type IdempotencyConflictError struct {
Err error
}
func (e *IdempotencyConflictError) Error() string {
if e == nil || e.Err == nil {
return "distributor idempotency conflict"
}
return e.Err.Error()
}
func (e *IdempotencyConflictError) Unwrap() error {
if e == nil {
return nil
}
return e.Err
}
type uploadClientFactory func(endpoint, token string, timeout time.Duration) (uploadClient, error)
type uploadClient interface {
UploadFiles(ctx context.Context, opts uploadFilesOptions) (uploadFilesResult, error)
}
type uploadFilesOptions struct {
BundleID string
IdempotencyKey string
SourcePath string
BundlePath string
}
type uploadFilesResult struct {
RunID string
Status string
}
func New(cfg config.DistributorNotifyConfig) *Client {
return newClient(cfg, newDistributorUploadClient)
}
func newClient(cfg config.DistributorNotifyConfig, factory uploadClientFactory) *Client {
if factory == nil {
factory = newDistributorUploadClient
}
return &Client{
Endpoint: cfg.Endpoint,
TokenEnv: cfg.TokenEnv,
Timeout: cfg.Timeout,
newUploadClient: factory,
}
}
func (c *Client) Upload(ctx context.Context, req UploadRequest) (UploadResult, error) {
if c == nil {
return UploadResult{}, fmt.Errorf("distributor client is nil")
}
if c.Endpoint == "" {
return UploadResult{}, fmt.Errorf("distributor endpoint is required")
}
if c.TokenEnv == "" {
return UploadResult{}, fmt.Errorf("distributor token environment variable is required")
}
if req.BundleID == "" {
return UploadResult{}, fmt.Errorf("distributor bundle id is required")
}
if req.IdempotencyKey == "" {
return UploadResult{}, fmt.Errorf("distributor idempotency key is required for bundle %q", req.BundleID)
}
if req.SourcePath == "" {
return UploadResult{}, fmt.Errorf("distributor source path is required for bundle %q", req.BundleID)
}
if req.BundlePath == "" {
return UploadResult{}, fmt.Errorf("distributor bundle path is required for bundle %q", req.BundleID)
}
if c.newUploadClient == nil {
return UploadResult{}, fmt.Errorf("distributor upload client factory is required for endpoint %q", c.Endpoint)
}
token := os.Getenv(c.TokenEnv)
if token == "" {
return UploadResult{}, fmt.Errorf("distributor token environment variable %q is not set", c.TokenEnv)
}
uploadClient, err := c.newUploadClient(c.Endpoint, token, c.Timeout)
if err != nil {
return UploadResult{}, fmt.Errorf("create distributor upload client for endpoint %q: %w", c.Endpoint, redactToken(err, token))
}
runCtx := ctx
if runCtx == nil {
runCtx = context.Background()
}
cancel := func() {}
if c.Timeout > 0 {
runCtx, cancel = context.WithTimeout(runCtx, c.Timeout)
}
defer cancel()
result, err := uploadClient.UploadFiles(runCtx, uploadFilesOptions{
BundleID: req.BundleID,
IdempotencyKey: req.IdempotencyKey,
SourcePath: req.SourcePath,
BundlePath: req.BundlePath,
})
if err != nil {
return UploadResult{}, wrapUploadError(err, uploadErrorContext{
Endpoint: c.Endpoint,
BundleID: req.BundleID,
IdempotencyKey: req.IdempotencyKey,
SourcePath: req.SourcePath,
BundlePath: req.BundlePath,
Token: token,
})
}
return UploadResult{
RunID: result.RunID,
Status: result.Status,
}, nil
}
type distributorUploadClient struct {
client *distributorupload.Client
}
func newDistributorUploadClient(endpoint, token string, timeout time.Duration) (uploadClient, error) {
httpClient := (*http.Client)(nil)
if timeout > 0 {
httpClient = &http.Client{Timeout: timeout}
}
client, err := distributorupload.NewClient(distributorupload.ClientOptions{
Endpoint: endpoint,
Token: token,
HTTPClient: httpClient,
})
if err != nil {
return nil, err
}
return distributorUploadClient{client: client}, nil
}
func (c distributorUploadClient) UploadFiles(ctx context.Context, opts uploadFilesOptions) (uploadFilesResult, error) {
result, err := c.client.UploadFiles(ctx, distributorupload.UploadFilesOptions{
ID: opts.BundleID,
IdempotencyKey: opts.IdempotencyKey,
Files: []distributorbundle.BundleFile{
{SourcePath: opts.SourcePath, Path: opts.BundlePath},
},
})
if err != nil {
return uploadFilesResult{}, err
}
return uploadFilesResult{
RunID: result.RunID,
Status: result.Status,
}, nil
}
type uploadErrorContext struct {
Endpoint string
BundleID string
IdempotencyKey string
SourcePath string
BundlePath string
Token string
}
func wrapUploadError(err error, ctx uploadErrorContext) error {
var conflict *distributorupload.IdempotencyConflictError
isConflict := errors.As(err, &conflict)
err = redactToken(err, ctx.Token)
if isConflict {
return &IdempotencyConflictError{
Err: fmt.Errorf("upload distributor bundle %q to endpoint %q with idempotency key %q from source %q as bundle path %q: idempotency conflict: %w", ctx.BundleID, ctx.Endpoint, ctx.IdempotencyKey, ctx.SourcePath, ctx.BundlePath, err),
}
}
return fmt.Errorf("upload distributor bundle %q to endpoint %q with idempotency key %q from source %q as bundle path %q: %w", ctx.BundleID, ctx.Endpoint, ctx.IdempotencyKey, ctx.SourcePath, ctx.BundlePath, err)
}
func redactToken(err error, token string) error {
if err == nil || token == "" {
return err
}
return errors.New(redactTokenString(err.Error(), token))
}
func redactTokenString(value, token string) string {
if token == "" {
return value
}
return strings.ReplaceAll(value, token, "[redacted]")
}

View File

@@ -0,0 +1,243 @@
package distributor
import (
"context"
"errors"
"fmt"
"strings"
"testing"
"time"
distributorupload "gitea.maximumdirect.net/eric/distributor/pkg/upload"
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
)
func TestUploadUsesConfiguredClientAndSingleFile(t *testing.T) {
cfg := config.Defaults().Notify.Distributor
cfg.Endpoint = "https://distributor.example.test"
cfg.TokenEnv = "DISTRIBUTOR_UPLOAD_TOKEN"
cfg.Timeout = 15 * time.Second
t.Setenv(cfg.TokenEnv, "secret-token")
factory := &fakeUploadFactory{
client: &fakeUploadClient{
result: uploadFilesResult{RunID: "run-123", Status: "accepted"},
},
}
client := newClient(cfg, factory.newClient)
result, err := client.Upload(context.Background(), UploadRequest{
BundleID: "weatherreporter.home.daily.run",
IdempotencyKey: "weatherreporter.home.daily.run",
SourcePath: "/tmp/report.md",
BundlePath: "daily.md",
})
if err != nil {
t.Fatalf("Upload() error = %v", err)
}
if result.RunID != "run-123" || result.Status != "accepted" {
t.Fatalf("result = %#v, want accepted run", result)
}
if factory.endpoint != cfg.Endpoint {
t.Fatalf("factory endpoint = %q, want %q", factory.endpoint, cfg.Endpoint)
}
if factory.token != "secret-token" {
t.Fatalf("factory token = %q, want secret-token", factory.token)
}
if factory.timeout != 15*time.Second {
t.Fatalf("factory timeout = %s, want 15s", factory.timeout)
}
got := factory.client.opts
if got.BundleID != "weatherreporter.home.daily.run" {
t.Fatalf("BundleID = %q, want weatherreporter.home.daily.run", got.BundleID)
}
if got.IdempotencyKey != "weatherreporter.home.daily.run" {
t.Fatalf("IdempotencyKey = %q, want weatherreporter.home.daily.run", got.IdempotencyKey)
}
if got.SourcePath != "/tmp/report.md" {
t.Fatalf("SourcePath = %q, want /tmp/report.md", got.SourcePath)
}
if got.BundlePath != "daily.md" {
t.Fatalf("BundlePath = %q, want daily.md", got.BundlePath)
}
}
func TestUploadRejectsMissingInputs(t *testing.T) {
cfg := config.Defaults().Notify.Distributor
t.Setenv(cfg.TokenEnv, "secret-token")
tests := []struct {
name string
mutate func(*Client, *UploadRequest)
wantErr string
}{
{
name: "Token",
mutate: func(c *Client, req *UploadRequest) {
t.Setenv(c.TokenEnv, "")
},
wantErr: "token environment variable",
},
{
name: "SourcePath",
mutate: func(c *Client, req *UploadRequest) {
req.SourcePath = ""
},
wantErr: "source path is required",
},
{
name: "BundlePath",
mutate: func(c *Client, req *UploadRequest) {
req.BundlePath = ""
},
wantErr: "bundle path is required",
},
{
name: "UploadClientFactory",
mutate: func(c *Client, req *UploadRequest) {
c.newUploadClient = nil
},
wantErr: "upload client factory is required",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Setenv(cfg.TokenEnv, "secret-token")
client := newClient(cfg, (&fakeUploadFactory{client: &fakeUploadClient{}}).newClient)
req := validUploadRequest()
tt.mutate(client, &req)
_, err := client.Upload(context.Background(), req)
if err == nil {
t.Fatal("Upload() error = nil, want error")
}
if !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("error = %q, want %q", err.Error(), tt.wantErr)
}
if strings.Contains(err.Error(), "secret-token") {
t.Fatalf("error = %q, want no token value", err.Error())
}
})
}
}
func TestUploadWrapsFactoryErrorWithoutToken(t *testing.T) {
cfg := config.Defaults().Notify.Distributor
cfg.Endpoint = "https://distributor.example.test"
t.Setenv(cfg.TokenEnv, "secret-token")
factory := &fakeUploadFactory{
err: fmt.Errorf("factory failed with secret-token"),
}
client := newClient(cfg, factory.newClient)
_, err := client.Upload(context.Background(), validUploadRequest())
if err == nil {
t.Fatal("Upload() error = nil, want error")
}
if strings.Contains(err.Error(), "secret-token") {
t.Fatalf("error = %q, want no token value", err.Error())
}
if !strings.Contains(err.Error(), cfg.Endpoint) {
t.Fatalf("error = %q, want endpoint context", err.Error())
}
}
func TestUploadWrapsUploadFailureWithContextWithoutToken(t *testing.T) {
cfg := config.Defaults().Notify.Distributor
cfg.Endpoint = "https://distributor.example.test"
t.Setenv(cfg.TokenEnv, "secret-token")
factory := &fakeUploadFactory{
client: &fakeUploadClient{err: fmt.Errorf("server rejected secret-token")},
}
client := newClient(cfg, factory.newClient)
req := validUploadRequest()
_, err := client.Upload(context.Background(), req)
if err == nil {
t.Fatal("Upload() error = nil, want error")
}
for _, want := range []string{cfg.Endpoint, req.BundleID, req.IdempotencyKey, req.SourcePath, req.BundlePath} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("error = %q, want context %q", err.Error(), want)
}
}
if strings.Contains(err.Error(), "secret-token") {
t.Fatalf("error = %q, want no token value", err.Error())
}
}
func TestUploadPreservesIdempotencyConflictDiagnosis(t *testing.T) {
cfg := config.Defaults().Notify.Distributor
cfg.Endpoint = "https://distributor.example.test"
t.Setenv(cfg.TokenEnv, "secret-token")
factory := &fakeUploadFactory{
client: &fakeUploadClient{
err: &distributorupload.IdempotencyConflictError{
HTTPError: distributorupload.HTTPError{
StatusCode: 409,
Status: "409 Conflict",
Message: "conflicting upload for secret-token",
},
},
},
}
client := newClient(cfg, factory.newClient)
_, err := client.Upload(context.Background(), validUploadRequest())
if err == nil {
t.Fatal("Upload() error = nil, want error")
}
var conflict *IdempotencyConflictError
if !errors.As(err, &conflict) {
t.Fatalf("Upload() error = %T %v, want IdempotencyConflictError", err, err)
}
if !strings.Contains(err.Error(), "idempotency conflict") {
t.Fatalf("error = %q, want idempotency conflict diagnosis", err.Error())
}
if strings.Contains(err.Error(), "secret-token") {
t.Fatalf("error = %q, want no token value", err.Error())
}
}
func validUploadRequest() UploadRequest {
return UploadRequest{
BundleID: "weatherreporter.home.daily.run",
IdempotencyKey: "weatherreporter.home.daily.run",
SourcePath: "/tmp/report.md",
BundlePath: "daily.md",
}
}
type fakeUploadFactory struct {
endpoint string
token string
timeout time.Duration
client *fakeUploadClient
err error
}
func (f *fakeUploadFactory) newClient(endpoint, token string, timeout time.Duration) (uploadClient, error) {
f.endpoint = endpoint
f.token = token
f.timeout = timeout
if f.err != nil {
return nil, f.err
}
return f.client, nil
}
type fakeUploadClient struct {
opts uploadFilesOptions
result uploadFilesResult
err error
}
func (c *fakeUploadClient) UploadFiles(ctx context.Context, opts uploadFilesOptions) (uploadFilesResult, error) {
c.opts = opts
if c.err != nil {
return uploadFilesResult{}, c.err
}
return c.result, nil
}