Skip to contentSkip to conversation

multipath systemd integration

PATCH In review Hannes Reinecke 2 participants 12 messages Nov 26, 2013 – Nov 27, 2013

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover multipath systemd integration

Hi Christophe,

here is now the patchset for systemd integration. As requested I've implemented a conditional compilation. Whenever systemd is installed on the system the systemd hooks are compiled in, or left out if not. I've also pulled the patch for the 'one-shot' mechanism as Ben Marzinski rightly pointed out issues with that one. The watchdog implementation has been clarified and tested; turned out to be a systemd issue where watchdog is only working properly on later versions of systemd.

Hannes Reinecke (9):
Clarify uxsock logging
Use system-provided regex implementation
multipathd: Add option '-s' to suppress timestamps
multipathd: switch to socket activation for systemd
multipathd: use sd_notify() to inform systemd
multipathd: Implement systemd watchdog integration
multipathd: enable core dumps for systemd
multipathd: Read environment variables from systemd
multipathd: measure path check time

 Makefile.inc                     |    6 +
 libmultipath/Makefile            |    8 +-
 libmultipath/debug.c             |   17 +-
 libmultipath/devmapper.c         |   16 +-
 libmultipath/regex.c             | 4032 --------------------------------------
 libmultipath/regex.h             |  252 ---
 libmultipath/uxsock.c            |   28 +-
 multipath/multipath.conf.5       |    7 +-
 multipathd/Makefile              |   26 +-
 multipathd/main.c                |  109 +-
 multipathd/multipathd.8          |   34 +-
 multipathd/multipathd.service    |   16 -
 multipathd/multipathd.service.in |   18 +
 multipathd/multipathd.socket     |    5 +
 multipathd/uxclnt.c              |    4 +-
 multipathd/uxlsnr.c              |    4 +-
 16 files changed, 228 insertions(+), 4354 deletions(-)
 delete mode 100644 libmultipath/regex.c
 delete mode 100644 libmultipath/regex.h
 delete mode 100644 multipathd/multipathd.service
 create mode 100644 multipathd/multipathd.service.in
 create mode 100644 multipathd/multipathd.socket
Signature · 1 line
1.8.1.4
#1 lore raw

PATCH 1/9 Clarify uxsock logging

Socket creation might fail on various stages, so print out a proper logging message.

---
 libmultipath/uxsock.c | 11 +++++++++--
 multipathd/uxclnt.c   |  4 +---
 multipathd/uxlsnr.c   |  4 +---
 3 files changed, 11 insertions(+), 8 deletions(-)
libmultipath/uxsock.c +9 -2
index fcad56e..e93edff 100644
@@ -19,6 +19,7 @@
 
 #include "memory.h"
 #include "uxsock.h"
+#include "debug.h"
 
 /*
  * connect to a unix domain socket
@@ -36,10 +37,12 @@ int ux_socket_connect(const char *name)
 
 	fd = socket(AF_LOCAL, SOCK_STREAM, 0);
 	if (fd == -1) {
+		condlog(3, "Couldn't create ux_socket, error %d", errno);
 		return -1;
 	}
 
 	if (connect(fd, (struct sockaddr *)&addr, len) == -1) {
+		condlog(3, "Couldn't connect to ux_socket, error %d", errno);
 		close(fd);
 		return -1;
 	}
@@ -57,7 +60,10 @@ int ux_socket_listen(const char *name)
 	struct sockaddr_un addr;
 
 	fd = socket(AF_LOCAL, SOCK_STREAM, 0);
-	if (fd == -1) return -1;
+	if (fd == -1) {
+		condlog(3, "Couldn't create ux_socket, error %d", errno);
+		return -1;
+	}
 
 	memset(&addr, 0, sizeof(addr));
 	addr.sun_family = AF_LOCAL;
@@ -66,15 +72,16 @@ int ux_socket_listen(const char *name)
 	strncpy(&addr.sun_path[1], name, len);
 
 	if (bind(fd, (struct sockaddr *)&addr, len) == -1) {
+		condlog(3, "Couldn't bind to ux_socket, error %d", errno);
 		close(fd);
 		return -1;
 	}
 
 	if (listen(fd, 10) == -1) {
+		condlog(3, "Couldn't listen to ux_socket, error %d", errno);
 		close(fd);
 		return -1;
 	}
-
 	return fd;
 }
 
multipathd/uxclnt.c +1 -3
index 3e4e381..e86be21 100644
@@ -121,10 +121,8 @@ int uxclnt(char * inbuf)
 	int fd;
 
 	fd = ux_socket_connect(DEFAULT_SOCKET);
-	if (fd == -1) {
-		perror("ux_socket_connect");
+	if (fd == -1)
 		exit(1);
-	}
 
 	if (inbuf)
 		process_req(fd, inbuf);
multipathd/uxlsnr.c +1 -3
index c0ddd8d..ed8e012 100644
@@ -112,10 +112,8 @@ void * uxsock_listen(int (*uxsock_trigger)(char *, char **, int *, void *),
 
 	ux_sock = ux_socket_listen(DEFAULT_SOCKET);
 
-	if (ux_sock == -1) {
-		condlog(0, "ux_socket_listen error");
+	if (ux_sock == -1)
 		exit(1);
-	}
 
 	pthread_cleanup_push(uxsock_cleanup, NULL);
 
Signature · 1 line
1.8.1.4
#2 lore raw

PATCH 2/9 Use system-provided regex implementation

In reply to Hannes Reinecke

There is zero value in carrying our own (old) regex implementation around; we're far better off using the system-provided one.

---
 libmultipath/Makefile |    2 +-
 libmultipath/regex.c  | 4032 -------------------------------------------------
 libmultipath/regex.h  |  252 ----
 3 files changed, 1 insertion(+), 4285 deletions(-)
 delete mode 100644 libmultipath/regex.c
 delete mode 100644 libmultipath/regex.h
libmultipath/Makefile +1 -1
index ae1d8a3..f760f24 100644
@@ -12,7 +12,7 @@ LIBDEPS = -lpthread -ldl -ldevmapper -ludev
 OBJS = memory.o parser.o vector.o devmapper.o callout.o \
        hwtable.o blacklist.o util.o dmparser.o config.o \
        structs.o discovery.o propsel.o dict.o \
-       pgpolicies.o debug.o regex.o defaults.o uevent.o \
+       pgpolicies.o debug.o defaults.o uevent.o \
        switchgroup.o uxsock.o print.o alias.o log_pthread.o \
        log.o configure.o structs_vec.o sysfs.o prio.o checkers.o \
        lock.o waiter.o file.o wwids.o prioritizers/alua_rtpg.o
libmultipath/regex.c deleted +0 -4032

2,992 lines not on this page · raw message

libmultipath/regex.h deleted +0 -252

252 lines not on this page · raw message

The rest of this diff is not shown. View the raw message.

Signature · 1 line
1.8.1.4

Part of the diff is not shown here. View the raw message.

#3 lore raw

PATCH 3/9 multipathd: Add option '-s' to suppress timestamps

In reply to Hannes Reinecke

systemd prefixes any messages to stdout with a timestamp, so it's quite pointless to do it ourself.

---
 libmultipath/debug.c     | 17 ++++++++++-------
 libmultipath/devmapper.c | 16 +++++++++-------
 multipathd/main.c        | 11 +++++++----
 multipathd/multipathd.8  |  7 +++++--
 4 files changed, 31 insertions(+), 20 deletions(-)
libmultipath/debug.c +10 -7
index d30517d..bad78a8 100644
@@ -21,15 +21,18 @@ void dlog (int sink, int prio, const char * fmt, ...)
 	thres = (conf) ? conf->verbosity : 0;
 
 	if (prio <= thres) {
-		if (!sink) {
-			time_t t = time(NULL);
-			struct tm *tb = localtime(&t);
-			char buff[16];
+		if (sink < 1) {
+			if (sink == 0) {
+				time_t t = time(NULL);
+				struct tm *tb = localtime(&t);
+				char buff[16];
 
-			strftime(buff, sizeof(buff), "%b %d %H:%M:%S", tb);
-			buff[sizeof(buff)-1] = '\0';
+				strftime(buff, sizeof(buff),
+					 "%b %d %H:%M:%S", tb);
+				buff[sizeof(buff)-1] = '\0';
 
-			fprintf(stdout, "%s | ", buff);
+				fprintf(stdout, "%s | ", buff);
+			}
 			vfprintf(stdout, fmt, ap);
 		}
 		else
libmultipath/devmapper.c +9 -7
index 4c8b923..71281e6 100644
@@ -71,15 +71,17 @@ dm_write_log (int level, const char *file, int line, const char *f, ...)
 		return;
 
 	va_start(ap, f);
-	if (!logsink) {
-		time_t t = time(NULL);
-		struct tm *tb = localtime(&t);
-		char buff[16];
+	if (logsink < 1) {
+		if (logsink == 0) {
+			time_t t = time(NULL);
+			struct tm *tb = localtime(&t);
+			char buff[16];
 
-		strftime(buff, sizeof(buff), "%b %d %H:%M:%S", tb);
-		buff[sizeof(buff)-1] = '\0';
+			strftime(buff, sizeof(buff), "%b %d %H:%M:%S", tb);
+			buff[sizeof(buff)-1] = '\0';
 
-		fprintf(stdout, "%s | ", buff);
+			fprintf(stdout, "%s | ", buff);
+		}
 		fprintf(stdout, "libdevmapper: %s(%i): ", file, line);
 		vfprintf(stdout, f, ap);
 		fprintf(stdout, "\n");
multipathd/main.c +7 -4
index 91d7bfc..78d9265 100644
@@ -1597,7 +1597,7 @@ child (void * param)
 	setup_thread_attr(&uevent_attr, 128 * 1024, 1);
 	setup_thread_attr(&waiter_attr, 32 * 1024, 1);
 
-	if (logsink) {
+	if (logsink == 1) {
 		setup_thread_attr(&log_attr, 64 * 1024, 0);
 		log_thread_start(&log_attr);
 		pthread_attr_destroy(&log_attr);
@@ -1749,7 +1749,7 @@ child (void * param)
 
 	condlog(2, "--------shut down-------");
 
-	if (logsink)
+	if (logsink == 1)
 		log_thread_stop();
 
 	/*
@@ -1849,7 +1849,7 @@ main (int argc, char *argv[])
 	if (!conf)
 		exit(1);
 
-	while ((arg = getopt(argc, argv, ":dv:k::")) != EOF ) {
+	while ((arg = getopt(argc, argv, ":dsv:k::")) != EOF ) {
 	switch(arg) {
 		case 'd':
 			logsink = 0;
@@ -1862,6 +1862,9 @@ main (int argc, char *argv[])
 
 			conf->verbosity = atoi(optarg);
 			break;
+		case 's':
+			logsink = -1;
+			break;
 		case 'k':
 			uxclnt(optarg);
 			exit(0);
@@ -1886,7 +1889,7 @@ main (int argc, char *argv[])
 		exit(0);
 	}
 
-	if (!logsink)
+	if (logsink < 1)
 		err = 0;
 	else
 		err = daemonize();
multipathd/multipathd.8 +5 -2
index ccc5f54..2aea150 100644
@@ -20,8 +20,11 @@ devmap reconfiguration, so that it can refresh its failed path list.
 .SH OPTIONS
 .TP
 .B \-d
-Forground Mode. Don't daemonize, and print all messages to stdout and stderr.
-.TP 
+Foreground Mode. Don't daemonize, and print all messages to stdout and stderr.
+.TP
+.B \-s
+Suppress timestamps. Do not prefix logging messages with a timestamp.
+.TP
 .B -v "level"
 Verbosity level. Print additional information while running multipathd. A  level of 0 means only print errors. A level of 3 or greater prints debugging information as well. 
 .TP
Signature · 1 line
1.8.1.4
#4 lore raw

PATCH 4/9 multipathd: switch to socket activation for systemd

In reply to Hannes Reinecke

multipathd already has a netlink socket for CLI commands, which can be used for socket activation.

---
 Makefile.inc                  |  6 ++++++
 libmultipath/Makefile         |  6 ++++++
 libmultipath/uxsock.c         | 17 +++++++++++++++++
 multipathd/Makefile           | 12 ++++++++++--
 multipathd/multipathd.service |  8 ++++----
 multipathd/multipathd.socket  |  5 +++++
 6 files changed, 48 insertions(+), 6 deletions(-)
 create mode 100644 multipathd/multipathd.socket
Makefile.inc +6 -0
index 92d2289..7186484 100644
@@ -21,6 +21,12 @@ ifndef LIB
 	endif
 endif
 
+ifndef SYSTEMD
+	ifeq ($(shell systemctl --version > /dev/null 2>&1 && echo 1), 1)
+		SYSTEMD = $(shell systemctl --version 2> /dev/null |  sed -n 's/systemd \([0-9]*\)/\1/p')
+	endif
+endif
+
 prefix      = 
 exec_prefix = $(prefix)
 bindir      = $(exec_prefix)/sbin
libmultipath/Makefile +6 -0
index f760f24..6364364 100644
@@ -8,6 +8,9 @@ SONAME=0
 DEVLIB = libmultipath.so
 LIBS = $(DEVLIB).$(SONAME)
 LIBDEPS = -lpthread -ldl -ldevmapper -ludev
+ifdef SYSTEMD
+	LIBDEPS += -lsystemd-daemon
+endif
 
 OBJS = memory.o parser.o vector.o devmapper.o callout.o \
        hwtable.o blacklist.o util.o dmparser.o config.o \
@@ -35,6 +38,9 @@ ifneq ($(strip $(LIBUDEV_API_RECVBUF)),0)
 	CFLAGS += -DLIBUDEV_API_RECVBUF
 endif
 
+ifdef SYSTEMD
+	CFLAGS += -DUSE_SYSTEMD=$(SYSTEMD)
+endif
 
 all: $(LIBS)
 
libmultipath/uxsock.c +17 -0
index e93edff..aff7a62 100644
@@ -16,6 +16,9 @@
 #include <sys/poll.h>
 #include <signal.h>
 #include <errno.h>
+#ifdef USE_SYSTEMD
+#include <systemd/sd-daemon.h>
+#endif
 
 #include "memory.h"
 #include "uxsock.h"
@@ -57,8 +60,22 @@ int ux_socket_connect(const char *name)
 int ux_socket_listen(const char *name)
 {
 	int fd, len;
+#ifdef USE_SYSTEMD
+	int num;
+#endif
 	struct sockaddr_un addr;
 
+#ifdef USE_SYSTEMD
+	num = sd_listen_fds(0);
+	if (num > 1) {
+		condlog(3, "sd_listen_fds returned %d fds", num);
+		return -1;
+	} else if (num == 1) {
+		fd = SD_LISTEN_FDS_START + 0;
+		condlog(3, "using fd %d from sd_listen_fds", fd);
+		return fd;
+	}
+#endif
 	fd = socket(AF_LOCAL, SOCK_STREAM, 0);
 	if (fd == -1) {
 		condlog(3, "Couldn't create ux_socket, error %d", errno);
multipathd/Makefile +10 -2
index b490c1d..3b9f2cf 100644
@@ -6,8 +6,12 @@ include ../Makefile.inc
 # basic flags setting
 #
 CFLAGS += -I$(multipathdir) -I$(mpathpersistdir)
-LDFLAGS += -lpthread -ldevmapper -lreadline -ludev -ldl \
-	   -L$(multipathdir) -lmultipath -L$(mpathpersistdir) -lmpathpersist
+LDFLAGS += -lpthread -ldevmapper -lreadline
+ifdef SYSTEMD
+	LDFLAGS += -lsystemd-daemon
+endif
+LDFLAGS += -ludev -ldl \
+	-L$(multipathdir) -lmultipath -L$(mpathpersistdir) -lmpathpersist
 
 #
 # debuging stuff
@@ -35,8 +39,11 @@ install:
 	$(INSTALL_PROGRAM) -d $(DESTDIR)$(bindir)
 	$(INSTALL_PROGRAM) -m 755 $(EXEC) $(DESTDIR)$(bindir)
 	$(INSTALL_PROGRAM) -d $(DESTDIR)$(rcdir)
+ifdef SYSTEMD
 	$(INSTALL_PROGRAM) -d $(DESTDIR)$(unitdir)
 	$(INSTALL_PROGRAM) -m 644 $(EXEC).service $(DESTDIR)$(unitdir)
+	$(INSTALL_PROGRAM) -m 644 $(EXEC).socket $(DESTDIR)$(unitdir)
+endif
 	$(INSTALL_PROGRAM) -d $(DESTDIR)$(mandir)
 	$(INSTALL_PROGRAM) -m 644 $(EXEC).8.gz $(DESTDIR)$(mandir)
 
@@ -45,6 +52,7 @@ uninstall:
 	rm -f $(DESTDIR)$(rcdir)/$(EXEC)
 	rm -f $(DESTDIR)$(mandir)/$(EXEC).8.gz
 	rm -f $(DESTDIR)$(unitdir)/$(EXEC).service
+	rm -f $(DESTDIR)$(unitdir)/$(EXEC).socket
 
 clean:
 	rm -f core *.o $(EXEC) *.gz
multipathd/multipathd.service +4 -4
index d6da067..ec38ef1 100644
@@ -6,11 +6,11 @@ DefaultDependencies=no
 Conflicts=shutdown.target
 
 [Service]
-Type=forking
-PIDFile=/var/run/multipathd.pid
-ExecStart=/sbin/multipathd
+ExecStartPre=/sbin/modprobe dm-multipath
+ExecStart=/sbin/multipathd -d -s
 ExecReload=/sbin/multipathd reconfigure
-#ExecStop=/path/to/scrip delete-me if not necessary
+ExecStop=/sbin/multipathd shutdown
 
 [Install]
 WantedBy=sysinit.target
+Also=multipathd.socket
multipathd/multipathd.socket new file +5 -0
new file mode 100644
index 0000000..3d4b6da
@@ -0,0 +1,5 @@
+[Socket]
+ListenStream=@/org/kernel/linux/storage/multipathd
+
+[Install]
+WantedBy=sockets.target
Signature · 1 line
1.8.1.4
#5 lore raw

PATCH 5/9 multipathd: use sd_notify() to inform systemd

In reply to Hannes Reinecke

Implement sd_notify() to inform systemd about our internal state. And we should be using the service type 'notify' so the systemd doesn't try to flood multipathd if it's still in discovery.

---
 multipathd/Makefile           |  3 +++
 multipathd/main.c             | 43 ++++++++++++++++++++++++++++++++++---------
 multipathd/multipathd.service |  3 ++-
 3 files changed, 39 insertions(+), 10 deletions(-)
multipathd/Makefile +3 -0
index 3b9f2cf..781122a 100644
@@ -6,6 +6,9 @@ include ../Makefile.inc
 # basic flags setting
 #
 CFLAGS += -I$(multipathdir) -I$(mpathpersistdir)
+ifdef SYSTEMD
+	CFLAGS += -DUSE_SYSTEMD=$(SYSTEMD)
+endif
 LDFLAGS += -lpthread -ldevmapper -lreadline
 ifdef SYSTEMD
 	LDFLAGS += -lsystemd-daemon
multipathd/main.c +34 -9

118 lines not on this page · raw message

multipathd/multipathd.service +2 -1
index ec38ef1..3874bcb 100644
@@ -6,10 +6,11 @@ DefaultDependencies=no
 Conflicts=shutdown.target
 
 [Service]
+Type=notify
+NotifyAccess=main
 ExecStartPre=/sbin/modprobe dm-multipath
 ExecStart=/sbin/multipathd -d -s
 ExecReload=/sbin/multipathd reconfigure
-ExecStop=/sbin/multipathd shutdown
 
 [Install]
 WantedBy=sysinit.target
Signature · 1 line
1.8.1.4
#6 lore raw

PATCH 6/9 multipathd: Implement systemd watchdog integration

In reply to Hannes Reinecke

In the past there have been several instances where multipathd would hang with the checkerloop as some path checker might not be able to return in time.
This patch now activates the watchdog feature from systemd to shutdown (and possibly restart) multipathd in these situations.
Due to a bug in systemd watchdog integration only works correctly with later version (> 206), so watchdog integration has been disabled per default on earlier implementations.

---
 multipath/multipath.conf.5       |  7 +++++--
 multipathd/Makefile              | 11 +++++++++--
 multipathd/main.c                | 15 ++++++++++++++-
 multipathd/multipathd.8          | 27 ++++++++++++++++++++++++++-
 multipathd/multipathd.service    | 17 -----------------
 multipathd/multipathd.service.in | 17 +++++++++++++++++
 6 files changed, 71 insertions(+), 23 deletions(-)
 delete mode 100644 multipathd/multipathd.service
 create mode 100644 multipathd/multipathd.service.in
multipath/multipath.conf.5 +5 -2
index 0fd3035..cf5bec0 100644
@@ -71,8 +71,11 @@ section recognizes the following keywords:
 .B polling_interval
 interval between two path checks in seconds. For properly functioning paths,
 the interval between checks will gradually increase to
-.B max_polling_interval;
-default is
+.B max_polling_interval.
+This value will be overridden by the
+.B WatchdogSec
+setting in the multipathd.service definition if systemd is used.
+Default is
 .I 5
 .TP
 .B max_polling_interval
multipathd/Makefile +9 -2
index 781122a..76b93fb 100644
@@ -32,12 +32,19 @@ OBJS = main.o pidfile.o uxlsnr.o uxclnt.o cli.o cli_handlers.o
 #
 # directives
 #
-all : $(EXEC)
+all : $(EXEC) $(EXEC).service
 
 $(EXEC): $(OBJS)
 	$(CC) $(CFLAGS) $(OBJS) $(LDFLAGS) -o $(EXEC)
 	$(GZIP) $(EXEC).8 > $(EXEC).8.gz
 
+$(EXEC).service: $(EXEC).service.in
+ifeq ($(shell test $(SYSTEMD) -ge 207 && echo 1), 1)
+	sed '/ExecReload.*/a WatchdogSec=5s\nRestart=on-failure\nRestartPreventExitStatus=1' $(EXEC).service.in > $(EXEC).service
+else
+	cp $(EXEC).service.in $(EXEC).service
+endif
+
 install:
 	$(INSTALL_PROGRAM) -d $(DESTDIR)$(bindir)
 	$(INSTALL_PROGRAM) -m 755 $(EXEC) $(DESTDIR)$(bindir)
@@ -58,5 +65,5 @@ uninstall:
 	rm -f $(DESTDIR)$(unitdir)/$(EXEC).socket
 
 clean:
-	rm -f core *.o $(EXEC) *.gz
+	rm -f core *.o $(EXEC) $(EXEC).service *.gz
 
multipathd/main.c +14 -1
index d8d1204..448ed39 100644
@@ -1288,6 +1288,7 @@ checkerloop (void *ap)
 		lock(vecs->lock);
 		pthread_testcancel();
 		condlog(4, "tick");
+		sd_notify(0, "WATCHDOG=1");
 
 		if (vecs->pathvec) {
 			vector_foreach_slot (vecs->pathvec, pp, i) {
@@ -1587,7 +1588,8 @@ child (void * param)
 	pthread_attr_t log_attr, misc_attr, uevent_attr;
 	struct vectors * vecs;
 	struct multipath * mpp;
-	int i;
+	char *envp;
+	int i, checkint;
 	int rc, pid_rc;
 
 	mlockall(MCL_CURRENT | MCL_FUTURE);
@@ -1663,6 +1665,17 @@ child (void * param)
 
 	conf->daemon = 1;
 	udev_set_sync_support(0);
+	envp = getenv("WATCHDOG_USEC");
+	if (envp && sscanf(envp, "%d", &checkint) == 1) {
+		/* Value is in microseconds */
+		checkint = checkint / 1000000;
+		if (checkint > conf->max_checkint)
+			conf->max_checkint = checkint;
+		conf->checkint = checkint;
+		condlog(3, "set checkint to %d max %d",
+			conf->checkint, conf->max_checkint);
+	}
+
 	/*
 	 * Start uevent listener early to catch events
 	 */
multipathd/multipathd.8 +26 -1
index 2aea150..2ccf003 100644
@@ -128,10 +128,35 @@ Restore queuing on multipahted map $map
 .B quit|exit
 End interactive session.
 
+.SH "SYSTEMD INTEGRATION"
+When compiled with systemd support two systemd service files are
+installed,
+.I multipathd.service
+and
+.I multipathd.socket
+The
+.I multipathd.socket
+service instructs systemd to intercept the CLI command socket, so
+that any call to the CLI interface will start-up the daemon if
+required.
+The
+.I multipathd.service
+file carries the definitions for controlling the multipath daemon.
+The daemon itself uses the
+.B sd_notify(3)
+interface to communicate with systemd. The multipath daemon supports
+the
+.I WatchdogSec=
+feature from systemd (see \fBsystemd.service(5)\fR).
+Please note that systemd prior to version 207 has issues which prevent
+the systemd-provided watchdog from working correctly. So for earlier
+versions of systemd watchdog support will not be enabled per default.
+
 .SH "SEE ALSO"
 .BR multipath (8)
 .BR kpartx (8)
-.BR hotplug (8)
+.BR sd_notify (3)
+.BR system.service (5)
 .SH "AUTHORS"
 .B multipathd
 was developed by Christophe Varoqui, <[email protected]> and others.
multipathd/multipathd.service deleted +0 -17
deleted file mode 100644
index 3874bcb..0000000
@@ -1,17 +0,0 @@
-[Unit]
-Description=Device-Mapper Multipath Device Controller
-Before=iscsi.service iscsid.service lvm2-activation-early.service
-After=syslog.target
-DefaultDependencies=no
-Conflicts=shutdown.target
-
-[Service]
-Type=notify
-NotifyAccess=main
-ExecStartPre=/sbin/modprobe dm-multipath
-ExecStart=/sbin/multipathd -d -s
-ExecReload=/sbin/multipathd reconfigure
-
-[Install]
-WantedBy=sysinit.target
-Also=multipathd.socket
multipathd/multipathd.service.in new file +17 -0
new file mode 100644
index 0000000..3874bcb
@@ -0,0 +1,17 @@
+[Unit]
+Description=Device-Mapper Multipath Device Controller
+Before=iscsi.service iscsid.service lvm2-activation-early.service
+After=syslog.target
+DefaultDependencies=no
+Conflicts=shutdown.target
+
+[Service]
+Type=notify
+NotifyAccess=main
+ExecStartPre=/sbin/modprobe dm-multipath
+ExecStart=/sbin/multipathd -d -s
+ExecReload=/sbin/multipathd reconfigure
+
+[Install]
+WantedBy=sysinit.target
+Also=multipathd.socket
Signature · 1 line
1.8.1.4
#7 lore raw

Re: [PATCH 6/9] multipathd: Implement systemd watchdog integration

Quoted 9 lines · Hannes Reinecke

In the past there have been several instances where multipathd would hang with the checkerloop as some path checker might not be able to return in time.
This patch now activates the watchdog feature from systemd to shutdown (and possibly restart) multipathd in these situations.
Due to a bug in systemd watchdog integration only works correctly with later version (> 206), so watchdog integration has been disabled per default on earlier implementations.

I'm still not sure what having multipath restarted gets us. Is the hope that on restart, multipath will simply be unable to access the path, and it will fail there quicker that the checker would? Otherwise, the checker will likely get stuck in the same place on the restart. Also, the checker can get stuck in uninterruptible sleep. In this case, systemd isn't going to be able to to restart multipathd until the issue has already cleared up.

Quoted 192 lines
---
 multipath/multipath.conf.5       |  7 +++++--
 multipathd/Makefile              | 11 +++++++++--
 multipathd/main.c                | 15 ++++++++++++++-
 multipathd/multipathd.8          | 27 ++++++++++++++++++++++++++-
 multipathd/multipathd.service    | 17 -----------------
 multipathd/multipathd.service.in | 17 +++++++++++++++++
 6 files changed, 71 insertions(+), 23 deletions(-)
 delete mode 100644 multipathd/multipathd.service
 create mode 100644 multipathd/multipathd.service.in
multipath/multipath.conf.5 +5 -2
index 0fd3035..cf5bec0 100644
@@ -71,8 +71,11 @@ section recognizes the following keywords:
 .B polling_interval
 interval between two path checks in seconds. For properly functioning paths,
 the interval between checks will gradually increase to
-.B max_polling_interval;
-default is
+.B max_polling_interval.
+This value will be overridden by the
+.B WatchdogSec
+setting in the multipathd.service definition if systemd is used.
+Default is
 .I 5
 .TP
 .B max_polling_interval
multipathd/Makefile +9 -2
index 781122a..76b93fb 100644
@@ -32,12 +32,19 @@ OBJS = main.o pidfile.o uxlsnr.o uxclnt.o cli.o cli_handlers.o
 #
 # directives
 #
-all : $(EXEC)
+all : $(EXEC) $(EXEC).service
 
 $(EXEC): $(OBJS)
 	$(CC) $(CFLAGS) $(OBJS) $(LDFLAGS) -o $(EXEC)
 	$(GZIP) $(EXEC).8 > $(EXEC).8.gz
 
+$(EXEC).service: $(EXEC).service.in
+ifeq ($(shell test $(SYSTEMD) -ge 207 && echo 1), 1)
+	sed '/ExecReload.*/a WatchdogSec=5s\nRestart=on-failure\nRestartPreventExitStatus=1' $(EXEC).service.in > $(EXEC).service
+else
+	cp $(EXEC).service.in $(EXEC).service
+endif
+
 install:
 	$(INSTALL_PROGRAM) -d $(DESTDIR)$(bindir)
 	$(INSTALL_PROGRAM) -m 755 $(EXEC) $(DESTDIR)$(bindir)
@@ -58,5 +65,5 @@ uninstall:
 	rm -f $(DESTDIR)$(unitdir)/$(EXEC).socket
 
 clean:
-	rm -f core *.o $(EXEC) *.gz
+	rm -f core *.o $(EXEC) $(EXEC).service *.gz
 
multipathd/main.c +14 -1
index d8d1204..448ed39 100644
@@ -1288,6 +1288,7 @@ checkerloop (void *ap)
 		lock(vecs->lock);
 		pthread_testcancel();
 		condlog(4, "tick");
+		sd_notify(0, "WATCHDOG=1");
 
 		if (vecs->pathvec) {
 			vector_foreach_slot (vecs->pathvec, pp, i) {
@@ -1587,7 +1588,8 @@ child (void * param)
 	pthread_attr_t log_attr, misc_attr, uevent_attr;
 	struct vectors * vecs;
 	struct multipath * mpp;
-	int i;
+	char *envp;
+	int i, checkint;
 	int rc, pid_rc;
 
 	mlockall(MCL_CURRENT | MCL_FUTURE);
@@ -1663,6 +1665,17 @@ child (void * param)
 
 	conf->daemon = 1;
 	udev_set_sync_support(0);
+	envp = getenv("WATCHDOG_USEC");
+	if (envp && sscanf(envp, "%d", &checkint) == 1) {
+		/* Value is in microseconds */
+		checkint = checkint / 1000000;
+		if (checkint > conf->max_checkint)
+			conf->max_checkint = checkint;
+		conf->checkint = checkint;
+		condlog(3, "set checkint to %d max %d",
+			conf->checkint, conf->max_checkint);
+	}
+
 	/*
 	 * Start uevent listener early to catch events
 	 */
multipathd/multipathd.8 +26 -1
index 2aea150..2ccf003 100644
@@ -128,10 +128,35 @@ Restore queuing on multipahted map $map
 .B quit|exit
 End interactive session.
 
+.SH "SYSTEMD INTEGRATION"
+When compiled with systemd support two systemd service files are
+installed,
+.I multipathd.service
+and
+.I multipathd.socket
+The
+.I multipathd.socket
+service instructs systemd to intercept the CLI command socket, so
+that any call to the CLI interface will start-up the daemon if
+required.
+The
+.I multipathd.service
+file carries the definitions for controlling the multipath daemon.
+The daemon itself uses the
+.B sd_notify(3)
+interface to communicate with systemd. The multipath daemon supports
+the
+.I WatchdogSec=
+feature from systemd (see \fBsystemd.service(5)\fR).
+Please note that systemd prior to version 207 has issues which prevent
+the systemd-provided watchdog from working correctly. So for earlier
+versions of systemd watchdog support will not be enabled per default.
+
 .SH "SEE ALSO"
 .BR multipath (8)
 .BR kpartx (8)
-.BR hotplug (8)
+.BR sd_notify (3)
+.BR system.service (5)
 .SH "AUTHORS"
 .B multipathd
 was developed by Christophe Varoqui, <[email protected]> and others.
multipathd/multipathd.service deleted +0 -17
deleted file mode 100644
index 3874bcb..0000000
@@ -1,17 +0,0 @@
-[Unit]
-Description=Device-Mapper Multipath Device Controller
-Before=iscsi.service iscsid.service lvm2-activation-early.service
-After=syslog.target
-DefaultDependencies=no
-Conflicts=shutdown.target
-
-[Service]
-Type=notify
-NotifyAccess=main
-ExecStartPre=/sbin/modprobe dm-multipath
-ExecStart=/sbin/multipathd -d -s
-ExecReload=/sbin/multipathd reconfigure
-
-[Install]
-WantedBy=sysinit.target
-Also=multipathd.socket
multipathd/multipathd.service.in new file +17 -0
new file mode 100644
index 0000000..3874bcb
@@ -0,0 +1,17 @@
+[Unit]
+Description=Device-Mapper Multipath Device Controller
+Before=iscsi.service iscsid.service lvm2-activation-early.service
+After=syslog.target
+DefaultDependencies=no
+Conflicts=shutdown.target
+
+[Service]
+Type=notify
+NotifyAccess=main
+ExecStartPre=/sbin/modprobe dm-multipath
+ExecStart=/sbin/multipathd -d -s
+ExecReload=/sbin/multipathd reconfigure
+
+[Install]
+WantedBy=sysinit.target
+Also=multipathd.socket
Signature · 1 line
1.8.1.4
#8 lore raw

Re: [PATCH 6/9] multipathd: Implement systemd watchdog integration

Quoted 11 lines · Benjamin Marzinski
Quoted 9 lines · Hannes Reinecke

In the past there have been several instances where multipathd would hang with the checkerloop as some path checker might not be able to return in time.
This patch now activates the watchdog feature from systemd to shutdown (and possibly restart) multipathd in these situations.
Due to a bug in systemd watchdog integration only works correctly with later version (> 206), so watchdog integration has been disabled per default on earlier implementations.

Quoted 8 lines

I'm still not sure what having multipath restarted gets us. Is the hope that on restart, multipath will simply be unable to access the path, and it will fail there quicker that the checker would? Otherwise, the checker will likely get stuck in the same place on the restart. Also, the checker can get stuck in uninterruptible sleep. In this case, systemd isn't going to be able to to restart multipathd until the issue has already cleared up.

Most cases I've come across where the checkerloop was hanging it was _not_ due to an uninterruptible sleep, but rather a bug in some odd cornercase. So there it definitely would make sense.

And if you don't like the 'restart' behaviour you can easily switch it off by just editing the service file.

In general the watchdog integration (with or without restart) is a _very_ useful thing, as multipathd hanging is a pain to debug on a customer site. If systemd notifies this debugging becomes _way_ easier.

Cheers,

Hannes

Signature · 4 lines
Dr. Hannes Reinecke		      zSeries & Storage
[email protected]			      +49 911 74053 688
SUSE LINUX Products GmbH, Maxfeldstr. 5, 90409 Nürnberg
GF: J. Hawn, J. Guild, F. Imendörffer, HRB 16746 (AG Nürnberg)
#9 lore raw

PATCH 7/9 multipathd: enable core dumps for systemd

In reply to Hannes Reinecke

Add 'LimitCORE' definition to the service file to enable core dumps when running under systemd.

---
 multipathd/multipathd.service.in | 1 +
 1 file changed, 1 insertion(+)
multipathd/multipathd.service.in +1 -0
index 3874bcb..be3ba3f 100644
@@ -8,6 +8,7 @@ Conflicts=shutdown.target
 [Service]
 Type=notify
 NotifyAccess=main
+LimitCORE=infinity
 ExecStartPre=/sbin/modprobe dm-multipath
 ExecStart=/sbin/multipathd -d -s
 ExecReload=/sbin/multipathd reconfigure
Signature · 1 line
1.8.1.4
#10 lore raw

PATCH 8/9 multipathd: Read environment variables from systemd

In reply to Hannes Reinecke

When systemd adjusts 'OOMScoreAdjust' and 'LimitNOFILE' we should take those settings and not try to adjust them again on our side.

---
 multipathd/main.c | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)
multipathd/main.c +11 -1
index 448ed39..b57fa5d 100644
@@ -1553,7 +1553,13 @@ set_oom_adj (void)
 #endif
 	FILE *fp;
 	struct stat st;
+	char *envp;
 
+	envp = getenv("OOMScoreAdjust");
+	if (envp) {
+		condlog(3, "Using systemd provided OOMScoreAdjust");
+		return;
+	}
 	do {
 		if (stat(file, &st) == 0){
 			fp = fopen(file, "w");
@@ -1630,7 +1636,11 @@ child (void * param)
 
 	setlogmask(LOG_UPTO(conf->verbosity + 3));
 
-	if (conf->max_fds) {
+	envp = getenv("LimitNOFILE");
+
+	if (envp) {
+		condlog(2,"Using systemd provided open fds limit of %s", envp);
+	} else if (conf->max_fds) {
 		struct rlimit fd_limit;
 
 		if (getrlimit(RLIMIT_NOFILE, &fd_limit) < 0) {
Signature · 1 line
1.8.1.4
#11 lore raw

PATCH 9/9 multipathd: measure path check time

In reply to Hannes Reinecke

Instrument code to measure path check time.

---
 multipathd/main.c | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)
multipathd/main.c +21 -7
index b57fa5d..67feb61 100644
@@ -1104,7 +1104,7 @@ int update_path_groups(struct multipath *mpp, struct vectors *vecs, int refresh)
 	return 0;
 }
 
-void
+int
 check_path (struct vectors * vecs, struct path * pp)
 {
 	int newstate;
@@ -1113,10 +1113,10 @@ check_path (struct vectors * vecs, struct path * pp)
 	int oldchkrstate = pp->chkrstate;
 
 	if (!pp->mpp)
-		return;
+		return 0;
 
 	if (pp->tick && --pp->tick)
-		return; /* don't check this path yet */
+		return 0; /* don't check this path yet */
 
 	/*
 	 * provision a next check soonest,
@@ -1131,7 +1131,7 @@ check_path (struct vectors * vecs, struct path * pp)
 	if (newstate == PATH_WILD || newstate == PATH_UNCHECKED) {
 		condlog(2, "%s: unusable path", pp->dev);
 		pathinfo(pp, conf->hwtable, 0);
-		return;
+		return 1;
 	}
 	/*
 	 * Async IO in flight. Keep the previous path state
@@ -1139,7 +1139,7 @@ check_path (struct vectors * vecs, struct path * pp)
 	 */
 	if (newstate == PATH_PENDING) {
 		pp->tick = 1;
-		return;
+		return 0;
 	}
 	/*
 	 * Synchronize with kernel state
@@ -1177,7 +1177,7 @@ check_path (struct vectors * vecs, struct path * pp)
 			pp->mpp->failback_tick = 0;
 
 			pp->mpp->stat_path_failures++;
-			return;
+			return 1;
 		}
 
 		if(newstate == PATH_UP || newstate == PATH_GHOST){
@@ -1262,6 +1262,7 @@ check_path (struct vectors * vecs, struct path * pp)
 			 (chkr_new_path_up && followover_should_failback(pp)))
 			switch_pathgroup(pp->mpp);
 	}
+	return 1;
 }
 
 static void *
@@ -1284,6 +1285,11 @@ checkerloop (void *ap)
 	}
 
 	while (1) {
+		struct timeval diff_time, start_time, end_time;
+		int num_paths = 0;
+
+		if (gettimeofday(&start_time, NULL) != 0)
+			start_time.tv_sec = 0;
 		pthread_cleanup_push(cleanup_lock, &vecs->lock);
 		lock(vecs->lock);
 		pthread_testcancel();
@@ -1292,7 +1298,7 @@ checkerloop (void *ap)
 
 		if (vecs->pathvec) {
 			vector_foreach_slot (vecs->pathvec, pp, i) {
-				check_path(vecs, pp);
+				num_paths += check_path(vecs, pp);
 			}
 		}
 		if (vecs->mpvec) {
@@ -1308,6 +1314,14 @@ checkerloop (void *ap)
 		}
 
 		lock_cleanup_pop(vecs->lock);
+		if (start_time.tv_sec &&
+		    gettimeofday(&end_time, NULL) == 0 &&
+		    num_paths) {
+			timersub(&end_time, &start_time, &diff_time);
+			condlog(3, "checked %d path%s in %lu.%06lu secs",
+				num_paths, num_paths > 1 ? "s" : "",
+				diff_time.tv_sec, diff_time.tv_usec);
+		}
 		sleep(1);
 	}
 	return NULL;
Signature · 1 line
1.8.1.4