Fix DoS vulnerability in libtirpc
Subsystems (2)
Dai Ngo
reviewerPATCH 1/1 Fix DoS vulnerability in libtirpc
Currently svc_run does not handle poll time and rendezvous_request does not handle EMFILE error returned from accept(2 as it used to. These two missing functionality were removed by commit b2c9430f46c4.
The effect of not handling poll timeout allows idle TCP conections to remain ESTABLISHED indefinitely. When the number of connections reaches the limit of the open file descriptors (ulimit -n) then accept(2) fails with EMFILE. Since there is no handling of EMFILE error this causes svc_run() to get in a tight loop calling accept(2). This resulting in the RPC service of svc_run is being down, it's no longer able to service any requests.
The below script, td.sh, with nc (nmap-ncat-7.80-3) can be used to take down the RPC service:
if [ $# -ne 3 ]; then
echo "$0: server dst_port conn_cnt" exit
fi
server=$1
dport=$2
conn_cnt=$3
echo "dport[$dport] server[$server] conn_cnt[$conn_cnt]"
pcnt=0
while [ $pcnt -lt $conn_cnt ]
do
nc -v --recv-only $server $dport &
pcnt=`expr $pcnt + 1`done
Fix by restoring code in svc_run to cleanup idle conncetions when poll(2) returns 0 (timeout) and in rendezvous_request to handle EMFILE error returned from accept(2).
--- src/libtirpc.map | 2 +- src/rpc_com.h | 2 ++ src/svc.c | 2 +- src/svc_run.c | 2 ++ src/svc_vc.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 54 insertions(+), 2 deletions(-)
src/libtirpc.map +1 -1
@@ -331,5 +331,5 @@ TIRPC_PRIVATE { global: __libc_clntudp_bufcreate; # private, but used by rpcbind: - __svc_clean_idle; svc_auth_none; libtirpc_set_debug; + __svc_destroy_idle; __svc_clean_idle; svc_auth_none; libtirpc_set_debug; };
src/rpc_com.h +2 -0
@@ -55,6 +55,7 @@ struct netbuf *__rpcb_findaddr_timed(rpcprog_t, rpcvers_t,bool_t __rpc_control (int ,void *);bool_t __svc_clean_idle (fd_set *,int ,bool_t ); +void __svc_destroy_idle (int ,bool_t );bool_t __xdrrec_setnonblock (XDR *,int );bool_t __xdrrec_getrec (XDR *,enum xprt_stat *,bool_t );void __xprt_unregister_unlocked (SVCXPRT *);
@@ -62,6 +63,7 @@ void __xprt_set_raddr(SVCXPRT *, const struct sockaddr_storage *);extern int __svc_maxrec; +extern SVCXPRT **__svc_xports; #ifdef __cplusplus }
src/svc.c +1 -1
@@ -57,7 +57,7 @@#define max (a, b) (a > b ? a : b) -static SVCXPRT **__svc_xports; +SVCXPRT **__svc_xports;int __svc_maxrec;/*
src/svc_run.c +2 -0
@@ -44,6 +44,7 @@#include "rpc_com.h" #include <sys/select.h> +void svc_run () {
@@ -86,6 +87,7 @@ svc_run()warn ("svc_run: - poll failed" );break ;case 0 : +__svc_destroy_idle (30 , FALSE);continue ;default :svc_getreq_poll (my_pollfd, i);
src/svc_vc.c +48 -0
63 lines not on this page · raw message
Signature · 1 line
2.26.2
Quoted 54 lines · Dai Ngo
Currently
svc_rundoes not handle poll time andrendezvous_requestdoes not handle EMFILE error returned from accept(2 as it used to. These two missing functionality were removed by commitb2c9430f46c4.The effect of not handling poll timeout allows idle TCP conections to remain ESTABLISHED indefinitely. When the number of connections reaches the limit of the open file descriptors (ulimit -n) then accept(2) fails with EMFILE. Since there is no handling of EMFILE error this causes
svc_run()to get in a tight loop calling accept(2). This resulting in the RPC service ofsvc_runis being down, it's no longer able to service any requests.The below script,
td.sh, with nc (nmap-ncat-7.80-3) can be used to take down the RPC service:if [ $# -ne 3 ]; then
echo "$0: serverdst_portconn_cnt"
exit
fi
server=$1
dport=$2
conn_cnt=$3
echo "dport[$dport] server[$server] conn_cnt[$conn_cnt]"pcnt=0
while [ $pcnt -lt $conn_cnt ]
donc -v --recv-only $server $dport & pcnt=`expr $pcnt + 1`done
Fix by restoring code in
svc_runto cleanup idle conncetions when poll(2) returns 0 (timeout) and inrendezvous_requestto handle EMFILE error returned from accept(2).Fixes b2c9430f46c4 Use poll() instead of select() in svc_run()Signed-off-by [email protected]---
src/libtirpc.map | 2 +- src/rpc_com.h | 2 ++ src/svc.c | 2 +- src/svc_run.c | 2 ++ src/svc_vc.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 54 insertions(+), 2 deletions(-)src/libtirpc.map +0
-0
src/libtirpc.map TIRPC_PRIVATE { · line 334
Quoted 5 lines
src/libtirpc.map +1
-1@@ -331,5 +331,5 @@ TIRPC_PRIVATE { global: __libc_clntudp_bufcreate; # private, but used by rpcbind: - __svc_clean_idle; svc_auth_none; libtirpc_set_debug; + __svc_destroy_idle; __svc_clean_idle; svc_auth_none; libtirpc_set_debug;
As I stated yesterday, as a reviewer I agree with everything in this patch except for the addition of __svc_destroy_idle to the library's private API list.
IMO the private __svc_clean_idle() API was a mistake we shouldn't make again. (That might be why Thorsten removed it years ago).
Thanks, Dai, for your work nailing this down!
Quoted 126 lines
src/libtirpc.map +0
-0};src/rpc_com.h +2
-0@@ -55,6 +55,7 @@ struct netbuf *__rpcb_findaddr_timed(rpcprog_t, rpcvers_t,bool_t __rpc_control (int ,void *);bool_t __svc_clean_idle (fd_set *,int ,bool_t ); +void __svc_destroy_idle (int ,bool_t );bool_t __xdrrec_setnonblock (XDR *,int );bool_t __xdrrec_getrec (XDR *,enum xprt_stat *,bool_t );void __xprt_unregister_unlocked (SVCXPRT *);@@ -62,6 +63,7 @@ void __xprt_set_raddr(SVCXPRT *, const struct sockaddr_storage *);extern int __svc_maxrec; +extern SVCXPRT **__svc_xports; #ifdef __cplusplus }src/svc.c +1
-1@@ -57,7 +57,7 @@#define max (a, b) (a > b ? a : b) -static SVCXPRT **__svc_xports; +SVCXPRT **__svc_xports;int __svc_maxrec;/* src/svc_run.c +1
-0@@ -44,6 +44,7 @@#include "rpc_com.h" #include <sys/select.h> +void
svc_run()
{src/svc_run.c +1
-0@@ -86,6 +87,7 @@ svc_run()warn ("svc_run: - poll failed" );break ;case 0 : +__svc_destroy_idle (30 , FALSE);continue ;default :svc_getreq_poll (my_pollfd, i);src/svc_vc.c +48
-0@@ -58,6 +58,7 @@#include <rpc/rpc.h> +#include "debug.h" #include "rpc_com.h" #include <getpeereid.h> @@ -337,6 +338,10 @@ again:if (sock <0 ) {if (errno ==EINTR ) goto again; +if (errno ==EMFILE || errno ==ENFILE ) { +/* remove least active fd */ +__svc_destroy_idle (0 , FALSE); + }return (FALSE); }/* @@ -820,3 +825,46 @@ __svc_clean_idle(fd_set *fds, int timeout, bool_t cleanblock) {return FALSE; } + +void +__svc_destroy_idle (int timeout,bool_t cleanblock) +{ +int i; +SVCXPRT *xprt, *least_active; +struct timeval tv, tdiff, tmax; +struct cf_conn *cd; + +gettimeofday (&tv,NULL ); + tmax.tv_sec = tmax.tv_usec =0 ; + least_active =NULL ; +rwlock_wrlock (&svc_fd_lock); + +for (i =0 ; i <= svc_max_pollfd; i++) { +if (svc_pollfd[i].fd ==-1 ) +continue ; + xprt = __svc_xports[i]; +if (xprt ==NULL || xprt->xp_ops ==NULL || + xprt->xp_ops->xp_recv != svc_vc_recv) +continue ; + cd = (struct cf_conn *)xprt->xp_p1; +if (!cleanblock && !cd->nonblock) +continue ; +if (timeout ==0 ) { +timersub (&tv, &cd->last_recv_time, &tdiff); + if (timercmp(&tdiff, &tmax, >)) { + tmax = tdiff; + least_active = xprt; + } +continue ; + } +if (tv.tv_sec - cd->last_recv_time.tv_sec > timeout) { +__xprt_unregister_unlocked (xprt); +__svc_vc_dodestroy (xprt); + } + } +if (timeout ==0 && least_active !=NULL ) { +__xprt_unregister_unlocked (least_active); +__svc_vc_dodestroy (least_active); + } +rwlock_unlock (&svc_fd_lock); +}Signature · 1 line
2.26.2
--
Chuck Lever