Skip to contentSkip to conversation

dlm_controld: support for mark and waitplock_recovery

PATCH New Alexander Aring 1 participant 5 messages Jul 9, 2020

lore
Share & tools
lore
#0 lore raw

v3 cover dlm_controld: support for mark and waitplock_recovery

Hi,

this patch series adds support for set the in-kernel socket skb mark value over dlm_controld. There exists two kinds of socket, one listen socket and multiple peer sockets. Both can be set via the dlm config file via "listen_mark" or multiple entries of:

node id=$NODEID mark=$MARK

whereas $NODEID is the corosync assigned nodeid. The given mark number can be hexadecimal or decimal.

Also it adds support to set the waitplock_recovery per cluster attribute by setting enable_waitplock_recover over file or argument configuration.

- Alex

changes since v3:
- add support for unsigned int default values
- changed default value of boolean arg enable_waitplock_recover from 1 to 0. dlm_controld was not using the default value anyway, because there exists a check to ignore default values and only set it when it's set. I changed it anyway to be consistent.
- Let mark per node config don't abort the whole node configuration if open fails (e.g. older kernel) instead we skip now to try to set the mark value. It will show up in the log that open fails.

changes since v2:
- remove leftover PRIu32 in nodeid configuration
- make unsigned int values also work with file configuration, was arg only before
- add support to set the waitplock_recovery switch via dlm_controld
- remove free function of node_config, may be necessary when implement some kind of NOHUP and reparse config file

Alexander Aring (4):
dlm_controld: add support for unsigned int values
dlm_controld: set listen skb mark setting
dlm_controld: add support for per nodeid configuration
dlm_controld: add support for waitplock_recovery switch

 dlm_controld/Makefile      |  3 +-
 dlm_controld/action.c      | 43 ++++++++++++++++++--
 dlm_controld/config.c      | 25 ++++++++++++
 dlm_controld/dlm.conf.5    | 23 +++++++++++
 dlm_controld/dlm_daemon.h  | 13 +++++-
 dlm_controld/main.c        | 74 ++++++++++++++++++++++------------
 dlm_controld/member.c      |  6 ++-
 dlm_controld/node_config.c | 82 ++++++++++++++++++++++++++++++++++++++
 dlm_controld/node_config.h | 31 ++++++++++++++
 9 files changed, 268 insertions(+), 32 deletions(-)
 create mode 100644 dlm_controld/node_config.c
 create mode 100644 dlm_controld/node_config.h
Signature · 1 line
2.26.2
#1 lore raw

PATCH v3 1/4 dlm_controld: add support for unsigned int values

This patch adds support for setting a unsigned integer value.

---
 dlm_controld/config.c     | 25 ++++++++++++++++
 dlm_controld/dlm_daemon.h |  6 ++++
 dlm_controld/main.c       | 60 ++++++++++++++++++++++-----------------
 3 files changed, 65 insertions(+), 26 deletions(-)
dlm_controld/config.c +25 -0
index 3ba8a53b..ec269168 100644
@@ -156,6 +156,19 @@ static void get_val_int(char *line, int *val_out)
 	*val_out = atoi(val);
 }
 
+static void get_val_uint(char *line, unsigned int *val_out)
+{
+	char key[MAX_LINE];
+	char val[MAX_LINE];
+	int rv;
+
+	rv = sscanf(line, "%[^=]=%s", key, val);
+	if (rv != 2)
+		return;
+
+	*val_out = strtoul(val, NULL, 0);
+}
+
 static void get_val_str(char *line, char *val_out)
 {
 	char key[MAX_LINE];
@@ -171,6 +184,7 @@ static void get_val_str(char *line, char *val_out)
 
 void set_opt_file(int update)
 {
+	unsigned int uval = 0;
 	struct dlm_option *o;
 	FILE *file;
 	char line[MAX_LINE];
@@ -238,6 +252,17 @@ void set_opt_file(int update)
 			log_debug("config file %s = %d cli_set %d use %d",
 				  o->name, o->file_int, o->cli_set, o->use_int);
 
+		} else if (o->req_arg == req_arg_uint) {
+			get_val_uint(line, &uval);
+
+			o->file_uint = uval;
+
+			if (!o->cli_set)
+				o->use_uint = o->file_uint;
+
+			log_debug("config file %s = %u cli_set %d use %u",
+				  o->name, o->file_uint, o->cli_set, o->use_uint);
+
 		} else if (o->req_arg == req_arg_bool) {
 			get_val_int(line, &val);
 
dlm_controld/dlm_daemon.h +6 -0
index 5b9a52da..47557a7c 100644
@@ -86,6 +86,7 @@ enum {
         req_arg_bool = 1,
         req_arg_int = 2,
         req_arg_str = 3,
+        req_arg_uint = 4,
 };
 
 enum {
@@ -125,22 +126,27 @@ struct dlm_option {
 
 	int use_int;
 	char *use_str;
+	unsigned int use_uint;
 
 	int default_int;
 	const char *default_str;
+	unsigned int default_uint;
 
 	int cli_set;
 	int cli_int;
 	char *cli_str;
+	unsigned int cli_uint;
 
 	int file_set;
 	int file_int;
 	char *file_str;
+	unsigned int file_uint;
 };
 
 EXTERN struct dlm_option dlm_options[dlm_options_max];
 #define opt(x) dlm_options[x].use_int
 #define opts(x) dlm_options[x].use_str
+#define optu(x) dlm_options[x].use_uint
 
 
 /* DLM_LOCKSPACE_LEN: maximum lockspace name length, from linux/dlmconstants.h.
dlm_controld/main.c +34 -26

188 lines not on this page · raw message

Signature · 1 line
2.26.2
#2 lore raw

PATCH v3 2/4 dlm_controld: set listen skb mark setting

In reply to Alexander Aring

This patch adds support to set the skb mark value for the in-kernel DLM listen socket.

---
 dlm_controld/action.c     | 2 ++
 dlm_controld/dlm.conf.5   | 2 ++
 dlm_controld/dlm_daemon.h | 1 +
 dlm_controld/main.c       | 5 +++++
 4 files changed, 10 insertions(+)
dlm_controld/action.c +2 -0
index ecd0d022..e901d555 100644
@@ -851,6 +851,8 @@ int setup_configfs_options(void)
 	    dlm_options[timewarn_ind].file_set)
 		set_configfs_cluster("timewarn_cs", NULL, opt(timewarn_ind));
 
+	set_configfs_cluster("mark", NULL, optu(mark_ind));
+
 	proto_name = opts(protocol_ind);
 	proto_num = -1;
 
dlm_controld/dlm.conf.5 +2 -0
index 09492176..771951d4 100644
@@ -40,6 +40,8 @@ protocol
 .br
 bind_all
 .br
+mark
+.br
 debug_logfile
 .br
 enable_plock
dlm_controld/dlm_daemon.h +1 -0
index 47557a7c..4aa2ff82 100644
@@ -97,6 +97,7 @@ enum {
         protocol_ind,
         debug_logfile_ind,
 	bind_all_ind,
+        mark_ind,
         enable_fscontrol_ind,
         enable_plock_ind,
         plock_debug_ind,
dlm_controld/main.c +5 -0
index 6129b8a6..22faa3d5 100644
@@ -1737,6 +1737,11 @@ static void set_opt_defaults(void)
 			0, NULL, 0,
 			""); /* do not advertise */
 
+	set_opt_default(mark_ind,
+			"mark", '\0', req_arg_uint,
+			0, NULL, 0,
+			"set mark value for the DLM in-kernel listen socket");
+
 	set_opt_default(debug_logfile_ind,
 			"debug_logfile", 'L', no_arg,
 			0, NULL, 0,
Signature · 1 line
2.26.2
#3 lore raw

PATCH v3 3/4 dlm_controld: add support for per nodeid configuration

In reply to Alexander Aring

This patch adds support to make a configuration per nodeid and key-value pairs. As example this patch will introduce the key mark to set via configfs comms per nodeid the SO_MARK socket option.

---
 dlm_controld/Makefile      |  3 +-
 dlm_controld/action.c      | 36 +++++++++++++++--
 dlm_controld/dlm.conf.5    | 19 +++++++++
 dlm_controld/dlm_daemon.h  |  5 ++-
 dlm_controld/main.c        |  4 ++
 dlm_controld/member.c      |  6 ++-
 dlm_controld/node_config.c | 82 ++++++++++++++++++++++++++++++++++++++
 dlm_controld/node_config.h | 31 ++++++++++++++
 8 files changed, 180 insertions(+), 6 deletions(-)
 create mode 100644 dlm_controld/node_config.c
 create mode 100644 dlm_controld/node_config.h
dlm_controld/Makefile +2 -1
index 6081cf8b..fbc8926c 100644
@@ -32,7 +32,8 @@ BIN_SOURCE = action.c \
              config.c \
              member.c \
              logging.c \
-             rbtree.c
+             rbtree.c \
+             node_config.c
 LIB_SOURCE = lib.c
 
 CFLAGS += -D_GNU_SOURCE -O2 -ggdb \
dlm_controld/action.c +33 -3
index e901d555..9e18d286 100644
@@ -570,15 +570,16 @@ static int add_configfs_base(void)
 	return rv;
 }
 
-int add_configfs_node(int nodeid, char *addr, int addrlen, int local)
+int add_configfs_node(int nodeid, char *addr, int addrlen, int local,
+		      uint32_t mark)
 {
 	char path[PATH_MAX];
 	char padded_addr[sizeof(struct sockaddr_storage)];
 	char buf[32];
 	int rv, fd;
 
-	log_debug("set_configfs_node %d %s local %d",
-		  nodeid, str_ip(addr), local);
+	log_debug("set_configfs_node %d %s local %d mark %" PRIu32,
+		  nodeid, str_ip(addr), local, mark);
 
 	/*
 	 * create comm dir for this node
@@ -639,6 +640,35 @@ int add_configfs_node(int nodeid, char *addr, int addrlen, int local)
 	}
 	close(fd);
 
+	/*
+	 * set skb mark for nodeid
+	 *
+	 * If open() fails we skip it because kernel doesn't support it.
+	 * It's not a required confiuration. It will show up in the log.
+	 */
+
+	memset(path, 0, PATH_MAX);
+	snprintf(path, PATH_MAX, "%s/%d/mark", COMMS_DIR, nodeid);
+
+	fd = open(path, O_WRONLY);
+	if (fd < 0) {
+		log_error("%s: open failed: %d", path, errno);
+		goto skip_non_required;
+	}
+
+	memset(buf, 0, sizeof(buf));
+	snprintf(buf, 32, "%" PRIu32, mark);
+
+	rv = do_write(fd, buf, strlen(buf));
+	if (rv < 0) {
+		log_error("%s: write failed: %d, %s", path, errno, buf);
+		close(fd);
+		return -1;
+	}
+	close(fd);
+
+skip_non_required:
+
 	/*
 	 * set local
 	 */
dlm_controld/dlm.conf.5 +19 -0
index 771951d4..1ce0c644 100644
@@ -392,6 +392,25 @@ master    foo node=2 weight=1
 In which case node 1 will master 2/3 of the total resources and node 2
 will master the other 1/3.
 
+.SS Node configuration
+
+Node configurations can be set by the node keyword followed of key-value
+pairs.
+
+.B Keys:
+
+.B mark
+The mark key can be used to set a specific mark value which is then used
+by the in-kernel DLM socket creation. This can be used to match for DLM
+specfic packets for e.g. routing.
+
+Example of setting a per socket value for nodeid 1 and a mark value
+of 42:
+
+node id=1 mark=42
+
+For local nodes this value doesn't have any effect.
+
 .SH SEE ALSO
 .BR dlm_controld (8),
 .BR dlm_tool (8)
dlm_controld/dlm_daemon.h +4 -1
index 4aa2ff82..0b4ae5f2 100644
@@ -40,6 +40,7 @@
 #include <sched.h>
 #include <signal.h>
 #include <dirent.h>
+#include <inttypes.h>
 #include <sys/sysmacros.h>
 
 #include <corosync/cpg.h>
@@ -48,6 +49,7 @@
 #include "libdlmcontrol.h"
 #include "dlm_controld.h"
 #include "fence_config.h"
+#include "node_config.h"
 #include "list.h"
 #include "rbtree.h"
 #include "linux_endian.h"
@@ -365,7 +367,8 @@ int set_sysfs_nodir(char *name, int val);
 int set_configfs_members(struct lockspace *ls, char *name,
 			 int new_count, int *new_members,
 			 int renew_count, int *renew_members);
-int add_configfs_node(int nodeid, char *addr, int addrlen, int local);
+int add_configfs_node(int nodeid, char *addr, int addrlen, int local,
+		      uint32_t mark);
 void del_configfs_node(int nodeid);
 void clear_configfs(void);
 int setup_configfs_options(void);
dlm_controld/main.c +4 -0
index 22faa3d5..8a5a2ad1 100644
@@ -2052,6 +2052,10 @@ int main(int argc, char **argv)
 	set_opt_cli(argc, argv);
 	set_opt_file(0);
 
+	rv = node_config_init(CONF_FILE_PATH);
+	if (rv)
+		return 1;
+
 	strcpy(fence_all_device.name, "fence_all");
 	strcpy(fence_all_device.agent, "dlm_stonith");
 	fence_all_device.unfence = 0;
dlm_controld/member.c +5 -1
index da3a1f5b..1d5bfa3d 100644
@@ -109,6 +109,7 @@ static void quorum_callback(quorum_handle_t h, uint32_t quorate,
 {
 	corosync_cfg_node_address_t addrs[MAX_NODE_ADDRESSES];
 	corosync_cfg_node_address_t *addrptr = addrs;
+	const struct node_config *nc;
 	cs_error_t err;
 	int i, j, num_addrs;
 	uint64_t now = monotime();
@@ -163,12 +164,15 @@ static void quorum_callback(quorum_handle_t h, uint32_t quorate,
 				continue;
 			}
 
+			nc = node_config_get(quorum_nodes[i]);
+
 			for (j = 0; j < num_addrs; j++) {
 				add_configfs_node(quorum_nodes[i],
 						  addrptr[j].address,
 						  addrptr[j].address_length,
 						  (quorum_nodes[i] ==
-						   our_nodeid));
+						   our_nodeid),
+						  nc->mark);
 			}
 		}
 	}
dlm_controld/node_config.c new file +82 -0

82 lines not on this page · raw message

dlm_controld/node_config.h new file +31 -0
new file mode 100644
index 00000000..f0a4c2a0
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2020 Red Hat, Inc.
+ *
+ * This copyrighted material is made available to anyone wishing to use,
+ * modify, copy, or redistribute it subject to the terms and conditions
+ * of the GNU General Public License v2 or (at your option) any later version.
+ */
+
+#ifndef _NODE_CONFIG_H_
+#define _NODE_CONFIG_H_
+
+#include <stdint.h>
+
+struct node_config {
+	uint32_t mark;
+};
+
+/*
+ * Returns -ENOENT if path does not exist or there is no
+ * config for nodeid in the file.
+ *
+ * Returns -EXYZ if there's a problem with the config.
+ *
+ * Returns 0 if a config was found with no problems.
+ */
+
+int node_config_init(const char *path);
+
+const struct node_config *node_config_get(int nodeid);
+
+#endif
Signature · 1 line
2.26.2
#4 lore raw

PATCH v3 4/4 dlm_controld: add support for waitplock_recovery switch

In reply to Alexander Aring

This patch adds support to set the cluster attribute waitplock_recovery via enable_waitplock_recover arg or config file attribute.

---
 dlm_controld/action.c     | 5 +++++
 dlm_controld/dlm.conf.5   | 2 ++
 dlm_controld/dlm_daemon.h | 1 +
 dlm_controld/main.c       | 5 +++++
 4 files changed, 13 insertions(+)
dlm_controld/action.c +5 -0
index 9e18d286..bc9c44f2 100644
@@ -881,6 +881,11 @@ int setup_configfs_options(void)
 	    dlm_options[timewarn_ind].file_set)
 		set_configfs_cluster("timewarn_cs", NULL, opt(timewarn_ind));
 
+	if (dlm_options[enable_waitplock_recovery_ind].cli_set ||
+	    dlm_options[enable_waitplock_recovery_ind].file_set)
+		set_configfs_cluster("waitplock_recovery", NULL,
+				     opt(enable_waitplock_recovery_ind));
+
 	set_configfs_cluster("mark", NULL, optu(mark_ind));
 
 	proto_name = opts(protocol_ind);
dlm_controld/dlm.conf.5 +2 -0
index 1ce0c644..e92dfc8e 100644
@@ -46,6 +46,8 @@ debug_logfile
 .br
 enable_plock
 .br
+enable_waitplock_recovery
+.br
 plock_debug
 .br
 plock_rate_limit
dlm_controld/dlm_daemon.h +1 -0
index 0b4ae5f2..ee21c256 100644
@@ -102,6 +102,7 @@ enum {
         mark_ind,
         enable_fscontrol_ind,
         enable_plock_ind,
+        enable_waitplock_recovery_ind,
         plock_debug_ind,
         plock_rate_limit_ind,
         plock_ownership_ind,
dlm_controld/main.c +5 -0
index 8a5a2ad1..8023f4b0 100644
@@ -1757,6 +1757,11 @@ static void set_opt_defaults(void)
 			1, NULL, 0,
 			"enable/disable posix lock support for cluster fs");
 
+	set_opt_default(enable_waitplock_recovery_ind,
+			"enable_waitplock_recovery", '\0', req_arg_bool,
+			0, NULL, 0,
+			"enable/disable posix lock to wait for dlm recovery after lock acquire");
+
 	set_opt_default(plock_debug_ind,
 			"plock_debug", 'P', no_arg,
 			0, NULL, 0,
Signature · 1 line
2.26.2