multipath systemd integration
Subsystems (2)
cover multipath systemd integration
Hi Christophe,
here is now the patchset for systemd integration. As requested I've implemented a conditional compilation. Whenever systemd is installed on the system the systemd hooks are compiled in, or left out if not. I've also pulled the patch for the 'one-shot' mechanism as Ben Marzinski rightly pointed out issues with that one. The watchdog implementation has been clarified and tested; turned out to be a systemd issue where watchdog is only working properly on later versions of systemd.
Hannes Reinecke (9):
Clarify uxsock logging
Use system-provided regex implementation
multipathd: Add option '-s' to suppress timestamps
multipathd: switch to socket activation for systemd
multipathd: use sd_notify() to inform systemd
multipathd: Implement systemd watchdog integration
multipathd: enable core dumps for systemd
multipathd: Read environment variables from systemd
multipathd: measure path check time
Makefile.inc | 6 + libmultipath/Makefile | 8 +- libmultipath/debug.c | 17 +- libmultipath/devmapper.c | 16 +- libmultipath/regex.c | 4032 -------------------------------------- libmultipath/regex.h | 252 --- libmultipath/uxsock.c | 28 +- multipath/multipath.conf.5 | 7 +- multipathd/Makefile | 26 +- multipathd/main.c | 109 +- multipathd/multipathd.8 | 34 +- multipathd/multipathd.service | 16 - multipathd/multipathd.service.in | 18 + multipathd/multipathd.socket | 5 + multipathd/uxclnt.c | 4 +- multipathd/uxlsnr.c | 4 +- 16 files changed, 228 insertions(+), 4354 deletions(-) delete mode 100644 libmultipath/regex.c delete mode 100644 libmultipath/regex.h delete mode 100644 multipathd/multipathd.service create mode 100644 multipathd/multipathd.service.in create mode 100644 multipathd/multipathd.socket
Signature · 1 line
1.8.1.4
PATCH 1/9 Clarify uxsock logging
Socket creation might fail on various stages, so print out a proper logging message.
--- libmultipath/uxsock.c | 11 +++++++++-- multipathd/uxclnt.c | 4 +--- multipathd/uxlsnr.c | 4 +--- 3 files changed, 11 insertions(+), 8 deletions(-)
libmultipath/uxsock.c +9 -2
@@ -19,6 +19,7 @@#include "memory.h" #include "uxsock.h" +#include "debug.h" /* * connect to a unix domain socket
@@ -36,10 +37,12 @@ int ux_socket_connect(const char *name) fd =socket (AF_LOCAL ,SOCK_STREAM ,0 );if (fd ==-1 ) { +condlog (3 ,"Couldn't create ux_socket, error %d" , errno);return -1 ; }if (connect (fd, (struct sockaddr *)&addr, len) ==-1 ) { +condlog (3 ,"Couldn't connect to ux_socket, error %d" , errno);close (fd);return -1 ; }
@@ -57,7 +60,10 @@ int ux_socket_listen(const char *name)struct sockaddr_un addr; fd =socket (AF_LOCAL ,SOCK_STREAM ,0 ); -if (fd ==-1 )return -1 ; +if (fd ==-1 ) { +condlog (3 ,"Couldn't create ux_socket, error %d" , errno); +return -1 ; + }memset (&addr,0 ,sizeof (addr)); addr.sun_family =AF_LOCAL ;
@@ -66,15 +72,16 @@ int ux_socket_listen(const char *name)strncpy (&addr.sun_path[1 ], name, len);if (bind (fd, (struct sockaddr *)&addr, len) ==-1 ) { +condlog (3 ,"Couldn't bind to ux_socket, error %d" , errno);close (fd);return -1 ; }if (listen (fd,10 ) ==-1 ) { +condlog (3 ,"Couldn't listen to ux_socket, error %d" , errno);close (fd);return -1 ; } -return fd; }
multipathd/uxclnt.c +1 -3
@@ -121,10 +121,8 @@ int uxclnt(char * inbuf)int fd; fd =ux_socket_connect (DEFAULT_SOCKET ); -if (fd ==-1 ) { -perror ("ux_socket_connect" ); +if (fd ==-1 )exit (1 ); - }if (inbuf)process_req (fd, inbuf);
multipathd/uxlsnr.c +1 -3
@@ -112,10 +112,8 @@ void * uxsock_listen(int (*uxsock_trigger)(char *, char **, int *, void *), ux_sock =ux_socket_listen (DEFAULT_SOCKET ); -if (ux_sock ==-1 ) { -condlog (0 ,"ux_socket_listen error" ); +if (ux_sock ==-1 )exit (1 ); - }pthread_cleanup_push (uxsock_cleanup,NULL );
Signature · 1 line
1.8.1.4
PATCH 2/9 Use system-provided regex implementation
In reply to Hannes ReineckeThere is zero value in carrying our own (old) regex implementation around; we're far better off using the system-provided one.
--- libmultipath/Makefile | 2 +- libmultipath/regex.c | 4032 ------------------------------------------------- libmultipath/regex.h | 252 ---- 3 files changed, 1 insertion(+), 4285 deletions(-) delete mode 100644 libmultipath/regex.c delete mode 100644 libmultipath/regex.h
libmultipath/Makefile +1 -1
@@ -12,7 +12,7 @@ LIBDEPS = -lpthread -ldl -ldevmapper -ludev OBJS = memory.o parser.o vector.o devmapper.o callout.o \ hwtable.o blacklist.o util.o dmparser.o config.o \ structs.o discovery.o propsel.o dict.o \ - pgpolicies.o debug.o regex.o defaults.o uevent.o \ + pgpolicies.o debug.o defaults.o uevent.o \ switchgroup.o uxsock.o print.o alias.o log_pthread.o \ log.o configure.o structs_vec.o sysfs.o prio.o checkers.o \ lock.o waiter.o file.o wwids.o prioritizers/alua_rtpg.o
libmultipath/regex.c deleted +0 -4032
2,992 lines not on this page · raw message
libmultipath/regex.h deleted +0 -252
252 lines not on this page · raw message
The rest of this diff is not shown. View the raw message.
Signature · 1 line
1.8.1.4
Part of the diff is not shown here. View the raw message.
PATCH 3/9 multipathd: Add option '-s' to suppress timestamps
In reply to Hannes Reineckesystemd prefixes any messages to stdout with a timestamp, so it's quite pointless to do it ourself.
--- libmultipath/debug.c | 17 ++++++++++------- libmultipath/devmapper.c | 16 +++++++++------- multipathd/main.c | 11 +++++++---- multipathd/multipathd.8 | 7 +++++-- 4 files changed, 31 insertions(+), 20 deletions(-)
libmultipath/debug.c +10 -7
@@ -21,15 +21,18 @@ void dlog (int sink, int prio, const char * fmt, ...) thres = (conf) ? conf->verbosity :0 ;if (prio <= thres) { -if (!sink) { -time_t t =time (NULL ); -struct tm *tb =localtime (&t); -char buff[16 ]; +if (sink <1 ) { +if (sink ==0 ) { +time_t t =time (NULL ); +struct tm *tb =localtime (&t); +char buff[16 ]; -strftime (buff,sizeof (buff),"%b %d %H:%M:%S" , tb); - buff[sizeof (buff)-1 ] ='\0' ; +strftime (buff,sizeof (buff), +"%b %d %H:%M:%S" , tb); + buff[sizeof (buff)-1 ] ='\0' ; -fprintf (stdout,"%s | " , buff); +fprintf (stdout,"%s | " , buff); + }vfprintf (stdout, fmt, ap); }else
libmultipath/devmapper.c +9 -7
@@ -71,15 +71,17 @@ dm_write_log (int level, const char *file, int line, const char *f, ...)return ;va_start (ap, f); -if (!logsink) { -time_t t =time (NULL ); -struct tm *tb =localtime (&t); -char buff[16 ]; +if (logsink <1 ) { +if (logsink ==0 ) { +time_t t =time (NULL ); +struct tm *tb =localtime (&t); +char buff[16 ]; -strftime (buff,sizeof (buff),"%b %d %H:%M:%S" , tb); - buff[sizeof (buff)-1 ] ='\0' ; +strftime (buff,sizeof (buff),"%b %d %H:%M:%S" , tb); + buff[sizeof (buff)-1 ] ='\0' ; -fprintf (stdout,"%s | " , buff); +fprintf (stdout,"%s | " , buff); + }fprintf (stdout,"libdevmapper: %s(%i): " , file, line);vfprintf (stdout, f, ap);fprintf (stdout,"\n" );
multipathd/main.c +7 -4
@@ -1597,7 +1597,7 @@ child (void * param)setup_thread_attr (&uevent_attr,128 *1024 ,1 );setup_thread_attr (&waiter_attr,32 *1024 ,1 ); -if (logsink) { +if (logsink ==1 ) {setup_thread_attr (&log_attr,64 *1024 ,0 );log_thread_start (&log_attr);pthread_attr_destroy (&log_attr);
@@ -1749,7 +1749,7 @@ child (void * param)condlog (2 ,"--------shut down-------" ); -if (logsink) +if (logsink ==1 )log_thread_stop ();/*
@@ -1849,7 +1849,7 @@ main (int argc, char *argv[])if (!conf)exit (1 ); -while ((arg =getopt (argc, argv,":dv:k::" )) !=EOF ) { +while ((arg =getopt (argc, argv,":dsv:k::" )) !=EOF ) {switch (arg) {case 'd' : logsink =0 ;
@@ -1862,6 +1862,9 @@ main (int argc, char *argv[]) conf->verbosity =atoi (optarg);break ; +case 's' : + logsink =-1 ; +break ;case 'k' :uxclnt (optarg);exit (0 );
@@ -1886,7 +1889,7 @@ main (int argc, char *argv[])exit (0 ); } -if (!logsink) +if (logsink <1 ) err =0 ;else err =daemonize ();
multipathd/multipathd.8 +5 -2
@@ -20,8 +20,11 @@ devmap reconfiguration, so that it can refresh its failed path list. .SH OPTIONS .TP .B \-d -Forground Mode. Don't daemonize, and print all messages to stdout and stderr. -.TP +Foreground Mode. Don't daemonize, and print all messages to stdout and stderr. +.TP +.B \-s +Suppress timestamps. Do not prefix logging messages with a timestamp. +.TP .B -v "level" Verbosity level. Print additional information while running multipathd. A level of 0 means only print errors. A level of 3 or greater prints debugging information as well. .TP
Signature · 1 line
1.8.1.4
PATCH 4/9 multipathd: switch to socket activation for systemd
In reply to Hannes Reineckemultipathd already has a netlink socket for CLI commands, which can be used for socket activation.
--- Makefile.inc | 6 ++++++ libmultipath/Makefile | 6 ++++++ libmultipath/uxsock.c | 17 +++++++++++++++++ multipathd/Makefile | 12 ++++++++++-- multipathd/multipathd.service | 8 ++++---- multipathd/multipathd.socket | 5 +++++ 6 files changed, 48 insertions(+), 6 deletions(-) create mode 100644 multipathd/multipathd.socket
Makefile.inc +6 -0
@@ -21,6 +21,12 @@ ifndef LIB endif endif +ifndef SYSTEMD + ifeq ($(shell systemctl --version > /dev/null 2>&1 && echo 1), 1) + SYSTEMD = $(shell systemctl --version 2> /dev/null | sed -n 's/systemd \([0-9]*\)/\1/p') + endif +endif + prefix = exec_prefix = $(prefix) bindir = $(exec_prefix)/sbin
libmultipath/Makefile +6 -0
@@ -8,6 +8,9 @@ SONAME=0 DEVLIB = libmultipath.so LIBS = $(DEVLIB).$(SONAME) LIBDEPS = -lpthread -ldl -ldevmapper -ludev +ifdef SYSTEMD + LIBDEPS += -lsystemd-daemon +endif OBJS = memory.o parser.o vector.o devmapper.o callout.o \ hwtable.o blacklist.o util.o dmparser.o config.o \
@@ -35,6 +38,9 @@ ifneq ($(strip $(LIBUDEV_API_RECVBUF)),0) CFLAGS += -DLIBUDEV_API_RECVBUF endif +ifdef SYSTEMD + CFLAGS += -DUSE_SYSTEMD=$(SYSTEMD) +endif all: $(LIBS)
libmultipath/uxsock.c +17 -0
@@ -16,6 +16,9 @@#include <sys/poll.h> #include <signal.h> #include <errno.h> +#ifdef USE_SYSTEMD +#include <systemd/sd-daemon.h> +#endif #include "memory.h" #include "uxsock.h"
@@ -57,8 +60,22 @@ int ux_socket_connect(const char *name)int ux_socket_listen (const char *name) {int fd, len; +#ifdef USE_SYSTEMD +int num; +#endif struct sockaddr_un addr; +#ifdef USE_SYSTEMD + num =sd_listen_fds (0 ); +if (num >1 ) { +condlog (3 ,"sd_listen_fds returned %d fds" , num); +return -1 ; + }else if (num ==1 ) { + fd =SD_LISTEN_FDS_START +0 ; +condlog (3 ,"using fd %d from sd_listen_fds" , fd); +return fd; + } +#endif fd =socket (AF_LOCAL ,SOCK_STREAM ,0 );if (fd ==-1 ) {condlog (3 ,"Couldn't create ux_socket, error %d" , errno);
multipathd/Makefile +10 -2
@@ -6,8 +6,12 @@ include ../Makefile.inc # basic flags setting # CFLAGS += -I$(multipathdir) -I$(mpathpersistdir) -LDFLAGS += -lpthread -ldevmapper -lreadline -ludev -ldl \ - -L$(multipathdir) -lmultipath -L$(mpathpersistdir) -lmpathpersist +LDFLAGS += -lpthread -ldevmapper -lreadline +ifdef SYSTEMD + LDFLAGS += -lsystemd-daemon +endif +LDFLAGS += -ludev -ldl \ + -L$(multipathdir) -lmultipath -L$(mpathpersistdir) -lmpathpersist # # debuging stuff
@@ -35,8 +39,11 @@ install: $(INSTALL_PROGRAM) -d $(DESTDIR)$(bindir) $(INSTALL_PROGRAM) -m 755 $(EXEC) $(DESTDIR)$(bindir) $(INSTALL_PROGRAM) -d $(DESTDIR)$(rcdir) +ifdef SYSTEMD $(INSTALL_PROGRAM) -d $(DESTDIR)$(unitdir) $(INSTALL_PROGRAM) -m 644 $(EXEC).service $(DESTDIR)$(unitdir) + $(INSTALL_PROGRAM) -m 644 $(EXEC).socket $(DESTDIR)$(unitdir) +endif $(INSTALL_PROGRAM) -d $(DESTDIR)$(mandir) $(INSTALL_PROGRAM) -m 644 $(EXEC).8.gz $(DESTDIR)$(mandir)
@@ -45,6 +52,7 @@ uninstall: rm -f $(DESTDIR)$(rcdir)/$(EXEC) rm -f $(DESTDIR)$(mandir)/$(EXEC).8.gz rm -f $(DESTDIR)$(unitdir)/$(EXEC).service + rm -f $(DESTDIR)$(unitdir)/$(EXEC).socket clean: rm -f core *.o $(EXEC) *.gz
multipathd/multipathd.service +4 -4
@@ -6,11 +6,11 @@ DefaultDependencies=no Conflicts=shutdown.target [Service] -Type=forking -PIDFile=/var/run/multipathd.pid -ExecStart=/sbin/multipathd +ExecStartPre=/sbin/modprobe dm-multipath +ExecStart=/sbin/multipathd -d -s ExecReload=/sbin/multipathd reconfigure -#ExecStop=/path/to/scrip delete-me if not necessary +ExecStop=/sbin/multipathd shutdown [Install] WantedBy=sysinit.target +Also=multipathd.socket
multipathd/multipathd.socket new file +5 -0
@@ -0,0 +1,5 @@ +[Socket] +ListenStream=@/org/kernel/linux/storage/multipathd + +[Install] +WantedBy=sockets.target
Signature · 1 line
1.8.1.4
PATCH 5/9 multipathd: use sd_notify() to inform systemd
In reply to Hannes ReineckeImplement sd_notify() to inform systemd about our internal state. And we should be using the service type 'notify' so the systemd doesn't try to flood multipathd if it's still in discovery.
--- multipathd/Makefile | 3 +++ multipathd/main.c | 43 ++++++++++++++++++++++++++++++++++--------- multipathd/multipathd.service | 3 ++- 3 files changed, 39 insertions(+), 10 deletions(-)
multipathd/Makefile +3 -0
@@ -6,6 +6,9 @@ include ../Makefile.inc # basic flags setting # CFLAGS += -I$(multipathdir) -I$(mpathpersistdir) +ifdef SYSTEMD + CFLAGS += -DUSE_SYSTEMD=$(SYSTEMD) +endif LDFLAGS += -lpthread -ldevmapper -lreadline ifdef SYSTEMD LDFLAGS += -lsystemd-daemon
multipathd/main.c +34 -9
118 lines not on this page · raw message
multipathd/multipathd.service +2 -1
@@ -6,10 +6,11 @@ DefaultDependencies=no Conflicts=shutdown.target [Service] +Type=notify +NotifyAccess=main ExecStartPre=/sbin/modprobe dm-multipath ExecStart=/sbin/multipathd -d -s ExecReload=/sbin/multipathd reconfigure -ExecStop=/sbin/multipathd shutdown [Install] WantedBy=sysinit.target
Signature · 1 line
1.8.1.4
PATCH 6/9 multipathd: Implement systemd watchdog integration
In reply to Hannes ReineckeIn the past there have been several instances where multipathd would hang with the checkerloop as some path checker might not be able to return in time.
This patch now activates the watchdog feature from systemd to shutdown (and possibly restart) multipathd in these situations.
Due to a bug in systemd watchdog integration only works correctly with later version (> 206), so watchdog integration has been disabled per default on earlier implementations.
--- multipath/multipath.conf.5 | 7 +++++-- multipathd/Makefile | 11 +++++++++-- multipathd/main.c | 15 ++++++++++++++- multipathd/multipathd.8 | 27 ++++++++++++++++++++++++++- multipathd/multipathd.service | 17 ----------------- multipathd/multipathd.service.in | 17 +++++++++++++++++ 6 files changed, 71 insertions(+), 23 deletions(-) delete mode 100644 multipathd/multipathd.service create mode 100644 multipathd/multipathd.service.in
multipath/multipath.conf.5 +5 -2
@@ -71,8 +71,11 @@ section recognizes the following keywords: .B polling_interval interval between two path checks in seconds. For properly functioning paths, the interval between checks will gradually increase to -.B max_polling_interval; -default is +.B max_polling_interval. +This value will be overridden by the +.B WatchdogSec +setting in the multipathd.service definition if systemd is used. +Default is .I 5 .TP .B max_polling_interval
multipathd/Makefile +9 -2
@@ -32,12 +32,19 @@ OBJS = main.o pidfile.o uxlsnr.o uxclnt.o cli.o cli_handlers.o # # directives # -all : $(EXEC) +all : $(EXEC) $(EXEC).service $(EXEC): $(OBJS) $(CC) $(CFLAGS) $(OBJS) $(LDFLAGS) -o $(EXEC) $(GZIP) $(EXEC).8 > $(EXEC).8.gz +$(EXEC).service: $(EXEC).service.in +ifeq ($(shell test $(SYSTEMD) -ge 207 && echo 1), 1) + sed '/ExecReload.*/a WatchdogSec=5s\nRestart=on-failure\nRestartPreventExitStatus=1' $(EXEC).service.in > $(EXEC).service +else + cp $(EXEC).service.in $(EXEC).service +endif + install: $(INSTALL_PROGRAM) -d $(DESTDIR)$(bindir) $(INSTALL_PROGRAM) -m 755 $(EXEC) $(DESTDIR)$(bindir)
@@ -58,5 +65,5 @@ uninstall: rm -f $(DESTDIR)$(unitdir)/$(EXEC).socket clean: - rm -f core *.o $(EXEC) *.gz + rm -f core *.o $(EXEC) $(EXEC).service *.gz
multipathd/main.c +14 -1
@@ -1288,6 +1288,7 @@ checkerloop (void *ap)lock (vecs->lock);pthread_testcancel ();condlog (4 ,"tick" ); +sd_notify (0 ,"WATCHDOG=1" );if (vecs->pathvec) { vector_foreach_slot (vecs->pathvec, pp, i) {
@@ -1587,7 +1588,8 @@ child (void * param)pthread_attr_t log_attr, misc_attr, uevent_attr;struct vectors * vecs;struct multipath * mpp; -int i; +char *envp; +int i, checkint;int rc, pid_rc;mlockall (MCL_CURRENT |MCL_FUTURE );
@@ -1663,6 +1665,17 @@ child (void * param) conf->daemon =1 ;udev_set_sync_support (0 ); + envp =getenv ("WATCHDOG_USEC" ); +if (envp &&sscanf (envp,"%d" , &checkint) ==1 ) { +/* Value is in microseconds */ + checkint = checkint /1000000 ; +if (checkint > conf->max_checkint) + conf->max_checkint = checkint; + conf->checkint = checkint; +condlog (3 ,"set checkint to %d max %d" , + conf->checkint, conf->max_checkint); + } +/* * Start uevent listener early to catch events */
multipathd/multipathd.8 +26 -1
@@ -128,10 +128,35 @@ Restore queuing on multipahted map $map .B quit|exit End interactive session. +.SH "SYSTEMD INTEGRATION" +When compiled with systemd support two systemd service files are +installed, +.I multipathd.service +and +.I multipathd.socket +The +.I multipathd.socket +service instructs systemd to intercept the CLI command socket, so +that any call to the CLI interface will start-up the daemon if +required. +The +.I multipathd.service +file carries the definitions for controlling the multipath daemon. +The daemon itself uses the +.B sd_notify(3) +interface to communicate with systemd. The multipath daemon supports +the +.I WatchdogSec= +feature from systemd (see \fBsystemd.service(5)\fR). +Please note that systemd prior to version 207 has issues which prevent +the systemd-provided watchdog from working correctly. So for earlier +versions of systemd watchdog support will not be enabled per default. + .SH "SEE ALSO" .BR multipath (8) .BR kpartx (8) -.BR hotplug (8) +.BR sd_notify (3) +.BR system.service (5) .SH "AUTHORS" .B multipathd was developed by Christophe Varoqui, <[email protected]> and others.
multipathd/multipathd.service deleted +0 -17
@@ -1,17 +0,0 @@ -[Unit] -Description=Device-Mapper Multipath Device Controller -Before=iscsi.service iscsid.service lvm2-activation-early.service -After=syslog.target -DefaultDependencies=no -Conflicts=shutdown.target - -[Service] -Type=notify -NotifyAccess=main -ExecStartPre=/sbin/modprobe dm-multipath -ExecStart=/sbin/multipathd -d -s -ExecReload=/sbin/multipathd reconfigure - -[Install] -WantedBy=sysinit.target -Also=multipathd.socket
multipathd/multipathd.service.in new file +17 -0
@@ -0,0 +1,17 @@ +[Unit] +Description=Device-Mapper Multipath Device Controller +Before=iscsi.service iscsid.service lvm2-activation-early.service +After=syslog.target +DefaultDependencies=no +Conflicts=shutdown.target + +[Service] +Type=notify +NotifyAccess=main +ExecStartPre=/sbin/modprobe dm-multipath +ExecStart=/sbin/multipathd -d -s +ExecReload=/sbin/multipathd reconfigure + +[Install] +WantedBy=sysinit.target +Also=multipathd.socket
Signature · 1 line
1.8.1.4
Benjamin Marzinski
maintainerRe: [PATCH 6/9] multipathd: Implement systemd watchdog integration
Quoted 9 lines · Hannes Reinecke
In the past there have been several instances where multipathd would hang with the checkerloop as some path checker might not be able to return in time.
This patch now activates the watchdog feature from systemd to shutdown (and possibly restart) multipathd in these situations.
Due to a bug in systemd watchdog integration only works correctly with later version (> 206), so watchdog integration has been disabled per default on earlier implementations.
I'm still not sure what having multipath restarted gets us. Is the hope that on restart, multipath will simply be unable to access the path, and it will fail there quicker that the checker would? Otherwise, the checker will likely get stuck in the same place on the restart. Also, the checker can get stuck in uninterruptible sleep. In this case, systemd isn't going to be able to to restart multipathd until the issue has already cleared up.
Quoted 192 lines
Signed-off-by Hannes Reinecke <[email protected]>--- multipath/multipath.conf.5 | 7 +++++-- multipathd/Makefile | 11 +++++++++-- multipathd/main.c | 15 ++++++++++++++- multipathd/multipathd.8 | 27 ++++++++++++++++++++++++++- multipathd/multipathd.service | 17 ----------------- multipathd/multipathd.service.in | 17 +++++++++++++++++ 6 files changed, 71 insertions(+), 23 deletions(-) delete mode 100644 multipathd/multipathd.service create mode 100644 multipathd/multipathd.service.inmultipath/multipath.conf.5 +5
-2@@ -71,8 +71,11 @@ section recognizes the following keywords: .B polling_interval interval between two path checks in seconds. For properly functioning paths, the interval between checks will gradually increase to -.B max_polling_interval; -default is +.B max_polling_interval. +This value will be overridden by the +.B WatchdogSec +setting in the multipathd.service definition if systemd is used. +Default is .I 5 .TP .B max_polling_intervalmultipathd/Makefile +9
-2@@ -32,12 +32,19 @@ OBJS = main.o pidfile.o uxlsnr.o uxclnt.o cli.o cli_handlers.o # # directives # -all : $(EXEC) +all : $(EXEC) $(EXEC).service $(EXEC): $(OBJS) $(CC) $(CFLAGS) $(OBJS) $(LDFLAGS) -o $(EXEC) $(GZIP) $(EXEC).8 > $(EXEC).8.gz +$(EXEC).service: $(EXEC).service.in +ifeq ($(shell test $(SYSTEMD) -ge 207 && echo 1), 1) + sed '/ExecReload.*/a WatchdogSec=5s\nRestart=on-failure\nRestartPreventExitStatus=1' $(EXEC).service.in > $(EXEC).service +else + cp $(EXEC).service.in $(EXEC).service +endif + install: $(INSTALL_PROGRAM) -d $(DESTDIR)$(bindir) $(INSTALL_PROGRAM) -m 755 $(EXEC) $(DESTDIR)$(bindir)@@ -58,5 +65,5 @@ uninstall: rm -f $(DESTDIR)$(unitdir)/$(EXEC).socket clean: - rm -f core *.o $(EXEC) *.gz + rm -f core *.o $(EXEC) $(EXEC).service *.gzmultipathd/main.c +14
-1@@ -1288,6 +1288,7 @@ checkerloop (void *ap)lock (vecs->lock);pthread_testcancel ();condlog (4 ,"tick" ); +sd_notify (0 ,"WATCHDOG=1" );if (vecs->pathvec) { vector_foreach_slot (vecs->pathvec, pp, i) {@@ -1587,7 +1588,8 @@ child (void * param)pthread_attr_t log_attr, misc_attr, uevent_attr;struct vectors * vecs;struct multipath * mpp; -int i; +char *envp; +int i, checkint;int rc, pid_rc;mlockall (MCL_CURRENT |MCL_FUTURE );@@ -1663,6 +1665,17 @@ child (void * param) conf->daemon =1 ;udev_set_sync_support (0 ); + envp =getenv ("WATCHDOG_USEC" ); +if (envp &&sscanf (envp,"%d" , &checkint) ==1 ) { +/* Value is in microseconds */ + checkint = checkint /1000000 ; +if (checkint > conf->max_checkint) + conf->max_checkint = checkint; + conf->checkint = checkint; +condlog (3 ,"set checkint to %d max %d" , + conf->checkint, conf->max_checkint); + } +/* * Start uevent listener early to catch events */ multipathd/multipathd.8 +26
-1@@ -128,10 +128,35 @@ Restore queuing on multipahted map $map .B quit|exit End interactive session. +.SH "SYSTEMD INTEGRATION" +When compiled with systemd support two systemd service files are +installed, +.I multipathd.service +and +.I multipathd.socket +The +.I multipathd.socket +service instructs systemd to intercept the CLI command socket, so +that any call to the CLI interface will start-up the daemon if +required. +The +.I multipathd.service +file carries the definitions for controlling the multipath daemon. +The daemon itself uses the +.B sd_notify(3) +interface to communicate with systemd. The multipath daemon supports +the +.I WatchdogSec= +feature from systemd (see \fBsystemd.service(5)\fR). +Please note that systemd prior to version 207 has issues which prevent +the systemd-provided watchdog from working correctly. So for earlier +versions of systemd watchdog support will not be enabled per default. + .SH "SEE ALSO" .BR multipath (8) .BR kpartx (8) -.BR hotplug (8) +.BR sd_notify (3) +.BR system.service (5) .SH "AUTHORS" .B multipathd was developed by Christophe Varoqui, <[email protected]> and others.multipathd/multipathd.service deleted +0
-17@@ -1,17 +0,0 @@ -[Unit] -Description=Device-Mapper Multipath Device Controller -Before=iscsi.service iscsid.service lvm2-activation-early.service -After=syslog.target -DefaultDependencies=no -Conflicts=shutdown.target - -[Service] -Type=notify -NotifyAccess=main -ExecStartPre=/sbin/modprobe dm-multipath -ExecStart=/sbin/multipathd -d -s -ExecReload=/sbin/multipathd reconfigure - -[Install] -WantedBy=sysinit.target -Also=multipathd.socketmultipathd/multipathd.service.in new file +17
-0@@ -0,0 +1,17 @@ +[Unit] +Description=Device-Mapper Multipath Device Controller +Before=iscsi.service iscsid.service lvm2-activation-early.service +After=syslog.target +DefaultDependencies=no +Conflicts=shutdown.target + +[Service] +Type=notify +NotifyAccess=main +ExecStartPre=/sbin/modprobe dm-multipath +ExecStart=/sbin/multipathd -d -s +ExecReload=/sbin/multipathd reconfigure + +[Install] +WantedBy=sysinit.target +Also=multipathd.socketSignature · 1 line
1.8.1.4
Re: [PATCH 6/9] multipathd: Implement systemd watchdog integration
Quoted 11 lines · Benjamin Marzinski
Quoted 9 lines · Hannes Reinecke
In the past there have been several instances where multipathd would hang with the checkerloop as some path checker might not be able to return in time.
This patch now activates the watchdog feature from systemd to shutdown (and possibly restart) multipathd in these situations.
Due to a bug in systemd watchdog integration only works correctly with later version (> 206), so watchdog integration has been disabled per default on earlier implementations.
Quoted 8 lines
I'm still not sure what having multipath restarted gets us. Is the hope that on restart, multipath will simply be unable to access the path, and it will fail there quicker that the checker would? Otherwise, the checker will likely get stuck in the same place on the restart. Also, the checker can get stuck in uninterruptible sleep. In this case, systemd isn't going to be able to to restart multipathd until the issue has already cleared up.
Most cases I've come across where the checkerloop was hanging it was _not_ due to an uninterruptible sleep, but rather a bug in some odd cornercase. So there it definitely would make sense.
And if you don't like the 'restart' behaviour you can easily switch it off by just editing the service file.
In general the watchdog integration (with or without restart) is a _very_ useful thing, as multipathd hanging is a pain to debug on a customer site. If systemd notifies this debugging becomes _way_ easier.
Cheers,
Hannes
Signature · 4 lines
Dr. Hannes Reinecke zSeries & Storage [email protected] +49 911 74053 688 SUSE LINUX Products GmbH, Maxfeldstr. 5, 90409 Nürnberg GF: J. Hawn, J. Guild, F. Imendörffer, HRB 16746 (AG Nürnberg)
PATCH 7/9 multipathd: enable core dumps for systemd
In reply to Hannes ReineckeAdd 'LimitCORE' definition to the service file to enable core dumps when running under systemd.
--- multipathd/multipathd.service.in | 1 + 1 file changed, 1 insertion(+)
multipathd/multipathd.service.in +1 -0
@@ -8,6 +8,7 @@ Conflicts=shutdown.target [Service] Type=notify NotifyAccess=main +LimitCORE=infinity ExecStartPre=/sbin/modprobe dm-multipath ExecStart=/sbin/multipathd -d -s ExecReload=/sbin/multipathd reconfigure
Signature · 1 line
1.8.1.4
PATCH 8/9 multipathd: Read environment variables from systemd
In reply to Hannes ReineckeWhen systemd adjusts 'OOMScoreAdjust' and 'LimitNOFILE' we should take those settings and not try to adjust them again on our side.
--- multipathd/main.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-)
multipathd/main.c +11 -1
@@ -1553,7 +1553,13 @@ set_oom_adj (void)#endif FILE *fp;struct stat st; +char *envp; + envp =getenv ("OOMScoreAdjust" ); +if (envp) { +condlog (3 ,"Using systemd provided OOMScoreAdjust" ); +return ; + } do { if (stat(file, &st) == 0){ fp = fopen(file, "w");
@@ -1630,7 +1636,11 @@ child (void * param)setlogmask (LOG_UPTO (conf->verbosity +3 )); -if (conf->max_fds) { + envp =getenv ("LimitNOFILE" ); + +if (envp) { +condlog (2 ,"Using systemd provided open fds limit of %s" , envp); + }else if (conf->max_fds) {struct rlimit fd_limit;if (getrlimit (RLIMIT_NOFILE , &fd_limit) <0 ) {
Signature · 1 line
1.8.1.4
PATCH 9/9 multipathd: measure path check time
In reply to Hannes ReineckeInstrument code to measure path check time.
--- multipathd/main.c | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-)
multipathd/main.c +21 -7
@@ -1104,7 +1104,7 @@ int update_path_groups(struct multipath *mpp, struct vectors *vecs, int refresh)return 0 ; } -void +int check_path (struct vectors * vecs,struct path * pp) {int newstate;
@@ -1113,10 +1113,10 @@ check_path (struct vectors * vecs, struct path * pp)int oldchkrstate = pp->chkrstate;if (!pp->mpp) -return ; +return 0 ;if (pp->tick && --pp->tick) -return ;/* don't check this path yet */ +return 0 ;/* don't check this path yet */ /* * provision a next check soonest,
@@ -1131,7 +1131,7 @@ check_path (struct vectors * vecs, struct path * pp)if (newstate ==PATH_WILD || newstate ==PATH_UNCHECKED ) {condlog (2 ,"%s: unusable path" , pp->dev);pathinfo (pp, conf->hwtable,0 ); -return ; +return 1 ; }/* * Async IO in flight. Keep the previous path state
@@ -1139,7 +1139,7 @@ check_path (struct vectors * vecs, struct path * pp)*/ if (newstate ==PATH_PENDING ) { pp->tick =1 ; -return ; +return 0 ; }/* * Synchronize with kernel state
@@ -1177,7 +1177,7 @@ check_path (struct vectors * vecs, struct path * pp) pp->mpp->failback_tick =0 ; pp->mpp->stat_path_failures++; -return ; +return 1 ; }if (newstate ==PATH_UP || newstate ==PATH_GHOST ){
@@ -1262,6 +1262,7 @@ check_path (struct vectors * vecs, struct path * pp) (chkr_new_path_up && followover_should_failback(pp)))switch_pathgroup (pp->mpp); } +return 1 ; } static void *
@@ -1284,6 +1285,11 @@ checkerloop (void *ap) }while (1 ) { +struct timeval diff_time, start_time, end_time; +int num_paths =0 ; + +if (gettimeofday (&start_time,NULL ) !=0 ) + start_time.tv_sec =0 ;pthread_cleanup_push (cleanup_lock, &vecs->lock);lock (vecs->lock);pthread_testcancel ();
@@ -1292,7 +1298,7 @@ checkerloop (void *ap)if (vecs->pathvec) { vector_foreach_slot (vecs->pathvec, pp, i) { -check_path (vecs, pp); + num_paths +=check_path (vecs, pp); } }if (vecs->mpvec) {
@@ -1308,6 +1314,14 @@ checkerloop (void *ap) }lock_cleanup_pop (vecs->lock); +if (start_time.tv_sec && +gettimeofday (&end_time,NULL ) ==0 && + num_paths) { +timersub (&end_time, &start_time, &diff_time); +condlog (3 ,"checked %d path%s in %lu.%06lu secs" , + num_paths, num_paths >1 ?"s" :"" , + diff_time.tv_sec, diff_time.tv_usec); + }sleep (1 ); }return NULL ;
Signature · 1 line
1.8.1.4