package common import ( "context" "fmt" "io" "net/http" ) // maxResponseBodyBytes is a hard safety limit on HTTP response bodies. // API responses should be small, so this protects us from accidental // or malicious large responses. const maxResponseBodyBytes = 2 << 21 // 4 MiB func FetchBody(ctx context.Context, client *http.Client, url, userAgent, accept string) ([]byte, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, err } if userAgent != "" { req.Header.Set("User-Agent", userAgent) } if accept != "" { req.Header.Set("Accept", accept) } res, err := client.Do(req) if err != nil { return nil, err } defer res.Body.Close() if res.StatusCode < 200 || res.StatusCode >= 300 { return nil, fmt.Errorf("HTTP %s", res.Status) } // Read at most maxResponseBodyBytes + 1 so we can detect overflow. limited := io.LimitReader(res.Body, maxResponseBodyBytes+1) b, err := io.ReadAll(limited) if err != nil { return nil, err } if len(b) == 0 { return nil, fmt.Errorf("empty response body") } if len(b) > maxResponseBodyBytes { return nil, fmt.Errorf("response body too large (>%d bytes)", maxResponseBodyBytes) } return b, nil }