gssd: Improve scalability by not waiting for child processes
Subsystems (2)
PATCH 1/1 gssd: Improve scalability by not waiting for child processes
Instead of waiting on every fork, which would
become a bottle neck during a mount storm, simply
set a SIGCHLD signal handler to do the wait on
the child process
--- utils/gssd/gssd.c | 18 ++++++++++++++++++ utils/gssd/gssd_proc.c | 11 ++--------- 2 files changed, 20 insertions(+), 9 deletions(-)
utils/gssd/gssd.c +18 -0
@@ -44,11 +44,13 @@#define _GNU_SOURCE#endif +#include <sys/types.h> #include <sys/param.h> #include <sys/socket.h> #include <sys/time.h> #include <sys/resource.h> #include <sys/inotify.h> +#include <sys/wait.h> #include <rpc/rpc.h> #include <netinet/in.h> #include <arpa/inet.h>
@@ -736,6 +738,21 @@ sig_die(int signal)printerr (1 ,"exiting on signal %d\n" , signal);exit (0 ); } +static void +sig_child (int signal) +{ +int err; +pid_t pid; + +/* Parent: just wait on child to exit and return */ +do { + pid =wait (&err); + }while (pid ==-1 && errno != -ECHILD ); + +if (WIFSIGNALED (err)) +printerr (0 ,"WARNING: forked child was killed" +"with signal %d\n" ,WTERMSIG (err)); +}static void usage (char *progname)
@@ -902,6 +919,7 @@ main(int argc, char *argv[])signal (SIGINT , sig_die);signal (SIGTERM , sig_die); +signal (SIGCHLD , sig_child);signal_set (&sighup_ev,SIGHUP , gssd_scan_cb,NULL );signal_add (&sighup_ev,NULL );event_set (&inotify_ev, inotify_fd,EV_READ |EV_PERSIST , gssd_inotify_cb,NULL );
utils/gssd/gssd_proc.c +2 -9
@@ -656,16 +656,9 @@ process_krb5_upcall(struct clnt_info *clp, uid_t uid, int fd, char *tgtname,/* fork() failed! */ printerr (0 ,"WARNING: unable to fork() to handle" "upcall: %s\n" ,strerror (errno)); -return ; +/* FALLTHROUGH */ default : -/* Parent: just wait on child to exit and return */ -do { - pid =wait (&err); - }while (pid ==-1 && errno != -ECHILD ); - -if (WIFSIGNALED (err)) -printerr (0 ,"WARNING: forked child was killed" -"with signal %d\n" ,WTERMSIG (err)); +/* Parent: Return and wait for the SIGCHLD */ return ; } no_fork:
Signature · 1 line
2.4.3
Quoted 72 lines · On Wed, 23 Sep 2015 17:20:50 -0400 Steve Dickson <[email protected]> wrote:
Instead of waiting on every fork, which would
become a bottle neck during a mount storm, simply
set a SIGCHLD signal handler to do the wait on
the child processSigned-off-by Steve Dickson <[email protected]>--- utils/gssd/gssd.c | 18 ++++++++++++++++++ utils/gssd/gssd_proc.c | 11 ++--------- 2 files changed, 20 insertions(+), 9 deletions(-)utils/gssd/gssd.c +18
-0@@ -44,11 +44,13 @@#define _GNU_SOURCE#endif +#include <sys/types.h> #include <sys/param.h> #include <sys/socket.h> #include <sys/time.h> #include <sys/resource.h> #include <sys/inotify.h> +#include <sys/wait.h> #include <rpc/rpc.h> #include <netinet/in.h> #include <arpa/inet.h> @@ -736,6 +738,21 @@ sig_die(int signal)printerr (1 ,"exiting on signal %d\n" , signal);exit (0 ); } +static void +sig_child (int signal) +{ +int err; +pid_t pid; + +/* Parent: just wait on child to exit and return */ +do { + pid =wait (&err); + }while (pid ==-1 && errno != -ECHILD ); + +if (WIFSIGNALED (err)) +printerr (0 ,"WARNING: forked child was killed" +"with signal %d\n" ,WTERMSIG (err)); +}static void usage (char *progname)@@ -902,6 +919,7 @@ main(int argc, char *argv[])signal (SIGINT , sig_die);signal (SIGTERM , sig_die); +signal (SIGCHLD , sig_child);signal_set (&sighup_ev,SIGHUP , gssd_scan_cb,NULL );signal_add (&sighup_ev,NULL );event_set (&inotify_ev, inotify_fd,EV_READ |EV_PERSIST , gssd_inotify_cb,NULL );utils/gssd/gssd_proc.c +1
-3@@ -656,16 +656,9 @@ process_krb5_upcall(struct clnt_info *clp, uid_t uid, int fd, char *tgtname,/* fork() failed! */ printerr (0 ,"WARNING: unable to fork() to handle" "upcall: %s\n" ,strerror (errno)); -return ; +/* FALLTHROUGH */ default : -/* Parent: just wait on child to exit and return */ - do {
utils/gssd/gssd_proc.c process_krb5_upcall(struct clnt_info *clp, uid_t uid, int fd, char *tgtname, · line 664
Quoted 10 lines
utils/gssd/gssd_proc.c +1
-6- pid =wait (&err); - }while (pid ==-1 && errno != -ECHILD ); - -if (WIFSIGNALED (err)) -printerr (0 ,"WARNING: forked child was killed" -"with signal %d\n" ,WTERMSIG (err)); +/* Parent: Return and wait for the SIGCHLD */ return ; } no_fork:
I was thinking that there was some reason that we couldn't do this -- that there were data structures that would get wiped if you got another upcall while the first was being processed. The forking should prevent that though, so I think this looks reasonable.
Quoted 91 lines · Jeff Layton
Quoted 72 lines · On Wed, 23 Sep 2015 17:20:50 -0400 Steve Dickson <[email protected]> wrote:
Instead of waiting on every fork, which would
become a bottle neck during a mount storm, simply
set a SIGCHLD signal handler to do the wait on
the child processSigned-off-by Steve Dickson <[email protected]>--- utils/gssd/gssd.c | 18 ++++++++++++++++++ utils/gssd/gssd_proc.c | 11 ++--------- 2 files changed, 20 insertions(+), 9 deletions(-)utils/gssd/gssd.c +18
-0@@ -44,11 +44,13 @@#define _GNU_SOURCE#endif +#include <sys/types.h> #include <sys/param.h> #include <sys/socket.h> #include <sys/time.h> #include <sys/resource.h> #include <sys/inotify.h> +#include <sys/wait.h> #include <rpc/rpc.h> #include <netinet/in.h> #include <arpa/inet.h> @@ -736,6 +738,21 @@ sig_die(int signal)printerr (1 ,"exiting on signal %d\n" , signal);exit (0 ); } +static void +sig_child (int signal) +{ +int err; +pid_t pid; + +/* Parent: just wait on child to exit and return */ +do { + pid =wait (&err); + }while (pid ==-1 && errno != -ECHILD ); + +if (WIFSIGNALED (err)) +printerr (0 ,"WARNING: forked child was killed" +"with signal %d\n" ,WTERMSIG (err)); +}static void usage (char *progname)@@ -902,6 +919,7 @@ main(int argc, char *argv[])signal (SIGINT , sig_die);signal (SIGTERM , sig_die); +signal (SIGCHLD , sig_child);signal_set (&sighup_ev,SIGHUP , gssd_scan_cb,NULL );signal_add (&sighup_ev,NULL );event_set (&inotify_ev, inotify_fd,EV_READ |EV_PERSIST , gssd_inotify_cb,NULL );utils/gssd/gssd_proc.c +1
-3@@ -656,16 +656,9 @@ process_krb5_upcall(struct clnt_info *clp, uid_t uid, int fd, char *tgtname,/* fork() failed! */ printerr (0 ,"WARNING: unable to fork() to handle" "upcall: %s\n" ,strerror (errno)); -return ; +/* FALLTHROUGH */ default : -/* Parent: just wait on child to exit and return */ - do {utils/gssd/gssd_proc.c process_krb5_upcall(struct clnt_info *clp, uid_t uid, int fd, char *tgtname, · line 664
Quoted 10 lines
utils/gssd/gssd_proc.c +1
-6- pid =wait (&err); - }while (pid ==-1 && errno != -ECHILD ); - -if (WIFSIGNALED (err)) -printerr (0 ,"WARNING: forked child was killed" -"with signal %d\n" ,WTERMSIG (err)); +/* Parent: Return and wait for the SIGCHLD */ return ; } no_fork:I was thinking that there was some reason that we couldn't do this -- that there were data structures that would get wiped if you got another upcall while the first was being processed. The forking should prevent that though, so I think this looks reasonable.
Quoted 2 lines
Acked-by Jeff Layton <[email protected]>
Self Nak... During my testing there was a large number zombie rpc.gssd process... I'm not sure why but they are there...
steved.
Quoted 94 lines · On Sat, 26 Sep 2015 09:55:21 -0400 Steve Dickson <[email protected]> wrote:
Quoted 91 lines · Jeff Layton
Quoted 72 lines · On Wed, 23 Sep 2015 17:20:50 -0400 Steve Dickson <[email protected]> wrote:
Instead of waiting on every fork, which would
become a bottle neck during a mount storm, simply
set a SIGCHLD signal handler to do the wait on
the child processSigned-off-by Steve Dickson <[email protected]>--- utils/gssd/gssd.c | 18 ++++++++++++++++++ utils/gssd/gssd_proc.c | 11 ++--------- 2 files changed, 20 insertions(+), 9 deletions(-)utils/gssd/gssd.c +18
-0@@ -44,11 +44,13 @@#define _GNU_SOURCE#endif +#include <sys/types.h> #include <sys/param.h> #include <sys/socket.h> #include <sys/time.h> #include <sys/resource.h> #include <sys/inotify.h> +#include <sys/wait.h> #include <rpc/rpc.h> #include <netinet/in.h> #include <arpa/inet.h> @@ -736,6 +738,21 @@ sig_die(int signal)printerr (1 ,"exiting on signal %d\n" , signal);exit (0 ); } +static void +sig_child (int signal) +{ +int err; +pid_t pid; + +/* Parent: just wait on child to exit and return */ +do { + pid =wait (&err); + }while (pid ==-1 && errno != -ECHILD ); + +if (WIFSIGNALED (err)) +printerr (0 ,"WARNING: forked child was killed" +"with signal %d\n" ,WTERMSIG (err)); +}static void usage (char *progname)@@ -902,6 +919,7 @@ main(int argc, char *argv[])signal (SIGINT , sig_die);signal (SIGTERM , sig_die); +signal (SIGCHLD , sig_child);signal_set (&sighup_ev,SIGHUP , gssd_scan_cb,NULL );signal_add (&sighup_ev,NULL );event_set (&inotify_ev, inotify_fd,EV_READ |EV_PERSIST , gssd_inotify_cb,NULL );utils/gssd/gssd_proc.c +1
-3@@ -656,16 +656,9 @@ process_krb5_upcall(struct clnt_info *clp, uid_t uid, int fd, char *tgtname,/* fork() failed! */ printerr (0 ,"WARNING: unable to fork() to handle" "upcall: %s\n" ,strerror (errno)); -return ; +/* FALLTHROUGH */ default : -/* Parent: just wait on child to exit and return */ - do {utils/gssd/gssd_proc.c process_krb5_upcall(struct clnt_info *clp, uid_t uid, int fd, char *tgtname, · line 664
Quoted 10 lines
utils/gssd/gssd_proc.c +1
-6- pid =wait (&err); - }while (pid ==-1 && errno != -ECHILD ); - -if (WIFSIGNALED (err)) -printerr (0 ,"WARNING: forked child was killed" -"with signal %d\n" ,WTERMSIG (err)); +/* Parent: Return and wait for the SIGCHLD */ return ; } no_fork:I was thinking that there was some reason that we couldn't do this -- that there were data structures that would get wiped if you got another upcall while the first was being processed. The forking should prevent that though, so I think this looks reasonable.
Quoted 6 lines
Quoted 2 lines
Acked-by Jeff Layton <[email protected]>Self Nak... During my testing there was a large number zombie rpc.gssd process... I'm not sure why but they are there...
steved.
It's probably what I mentioned in the other mail.
If you get several children exiting in quick succession then you may only have one SIGCHLD pending. What you really want to do there is to have your SIGCHLD handler reap as many children as it can in a non-blocking fashion and then return. Even better would be to utilize the event loop for handling the signal...
Signature · 1 line
Jeff Layton <[email protected]>
Quoted 39 lines · On Wed, 23 Sep 2015 17:20:50 -0400 Steve Dickson <[email protected]> wrote:
Instead of waiting on every fork, which would
become a bottle neck during a mount storm, simply
set a SIGCHLD signal handler to do the wait on
the child processSigned-off-by Steve Dickson <[email protected]>--- utils/gssd/gssd.c | 18 ++++++++++++++++++ utils/gssd/gssd_proc.c | 11 ++--------- 2 files changed, 20 insertions(+), 9 deletions(-)utils/gssd/gssd.c +8
-0@@ -44,11 +44,13 @@#define _GNU_SOURCE#endif +#include <sys/types.h> #include <sys/param.h> #include <sys/socket.h> #include <sys/time.h> #include <sys/resource.h> #include <sys/inotify.h> +#include <sys/wait.h> #include <rpc/rpc.h> #include <netinet/in.h> #include <arpa/inet.h> @@ -736,6 +738,21 @@ sig_die(int signal)printerr (1 ,"exiting on signal %d\n" , signal);exit (0 ); } +static void +sig_child (int signal) +{ +int err; +pid_t pid; +
utils/gssd/gssd.c sig_die(int signal) · line 756
Quoted 10 lines
utils/gssd/gssd.c +9
-0+/* Parent: just wait on child to exit and return */ +do { + pid =wait (&err); + }while (pid ==-1 && errno != -ECHILD ); + +if (WIFSIGNALED (err)) +printerr (0 ,"WARNING: forked child was killed" +"with signal %d\n" ,WTERMSIG (err)); +}
That said, there is a problem here. You're going to get a SIGCHLD for each child that exits and multiples could exit at nearly the same time, so some of the SIGCHLDs can get "lost". This probably needs to keep reaping children (in a non-blocking way) until there aren't any more that have exited.
Also, given that gssd is based around libevent, you may be better off adding a libevent signal event handler instead of doing it with a "raw" signal handler. That way you won't block upcalls that are in the middle of running just to reap child processes.
Quoted 33 lines
utils/gssd/gssd.c +1
-0static void usage (char *progname)@@ -902,6 +919,7 @@ main(int argc, char *argv[])signal (SIGINT , sig_die);signal (SIGTERM , sig_die); +signal (SIGCHLD , sig_child);signal_set (&sighup_ev,SIGHUP , gssd_scan_cb,NULL );signal_add (&sighup_ev,NULL );event_set (&inotify_ev, inotify_fd,EV_READ |EV_PERSIST , gssd_inotify_cb,NULL );utils/gssd/gssd_proc.c +2
-9@@ -656,16 +656,9 @@ process_krb5_upcall(struct clnt_info *clp, uid_t uid, int fd, char *tgtname,/* fork() failed! */ printerr (0 ,"WARNING: unable to fork() to handle" "upcall: %s\n" ,strerror (errno)); -return ; +/* FALLTHROUGH */ default : -/* Parent: just wait on child to exit and return */ -do { - pid =wait (&err); - }while (pid ==-1 && errno != -ECHILD ); - -if (WIFSIGNALED (err)) -printerr (0 ,"WARNING: forked child was killed" -"with signal %d\n" ,WTERMSIG (err)); +/* Parent: Return and wait for the SIGCHLD */ return ; } no_fork:
Signature · 1 line
Jeff Layton <[email protected]>
* Steve Dickson:
Quoted 5 lines
utils/gssd/gssd.c +5
-0+static void +sig_child (int signal) +{ +int err; +pid_t pid;
utils/gssd/gssd.c sig_die(int signal) · line 755
Quoted 10 lines
utils/gssd/gssd.c +10
-0+ +/* Parent: just wait on child to exit and return */ +do { + pid =wait (&err); + }while (pid ==-1 && errno != -ECHILD ); + +if (WIFSIGNALED (err)) +printerr (0 ,"WARNING: forked child was killed" +"with signal %d\n" ,WTERMSIG (err)); +}
prinerr calls vfprintf or vsyslog. Neither is safe to use in signal handlers, so you need to log this message in some other way.
Florian
Quoted 12 lines · Florian Weimer
* Steve Dickson:
Quoted 10 lines
utils/gssd/gssd.c +10
-0+static void +sig_child (int signal) +{ +int err; +pid_t pid; + +/* Parent: just wait on child to exit and return */ +do { + pid =wait (&err); + }while (pid ==-1 && errno != -ECHILD );
Quoted 8 lines
utils/gssd/gssd.c sig_die(int signal) · line 755
Quoted 5 lines
utils/gssd/gssd.c +5
-0+ +if (WIFSIGNALED (err)) +printerr (0 ,"WARNING: forked child was killed" +"with signal %d\n" ,WTERMSIG (err)); +}prinerr calls vfprintf or vsyslog. Neither is safe to use in signal handlers, so you need to log this message in some other way.
Good point... but this patch was self NAK-ed due to it leaving zombie processes during my testing.
steved.