dlm_controld: support for mark and waitplock_recovery
v3 cover dlm_controld: support for mark and waitplock_recovery
Hi,
this patch series adds support for set the in-kernel socket skb mark value over dlm_controld. There exists two kinds of socket, one listen socket and multiple peer sockets. Both can be set via the dlm config file via "listen_mark" or multiple entries of:
node id=$NODEID mark=$MARK
whereas $NODEID is the corosync assigned nodeid. The given mark number can be hexadecimal or decimal.
Also it adds support to set the waitplock_recovery per cluster attribute by setting enable_waitplock_recover over file or argument configuration.
- Alex
changes since v3:
- add support for unsigned int default values
- changed default value of boolean arg enable_waitplock_recover from 1 to 0. dlm_controld was not using the default value anyway, because there exists a check to ignore default values and only set it when it's set. I changed it anyway to be consistent.
- Let mark per node config don't abort the whole node configuration if open fails (e.g. older kernel) instead we skip now to try to set the mark value. It will show up in the log that open fails.
changes since v2:
- remove leftover PRIu32 in nodeid configuration
- make unsigned int values also work with file configuration, was arg only before
- add support to set the waitplock_recovery switch via dlm_controld
- remove free function of node_config, may be necessary when implement some kind of NOHUP and reparse config file
Alexander Aring (4):
dlm_controld: add support for unsigned int values
dlm_controld: set listen skb mark setting
dlm_controld: add support for per nodeid configuration
dlm_controld: add support for waitplock_recovery switch
dlm_controld/Makefile | 3 +- dlm_controld/action.c | 43 ++++++++++++++++++-- dlm_controld/config.c | 25 ++++++++++++ dlm_controld/dlm.conf.5 | 23 +++++++++++ dlm_controld/dlm_daemon.h | 13 +++++- dlm_controld/main.c | 74 ++++++++++++++++++++++------------ dlm_controld/member.c | 6 ++- dlm_controld/node_config.c | 82 ++++++++++++++++++++++++++++++++++++++ dlm_controld/node_config.h | 31 ++++++++++++++ 9 files changed, 268 insertions(+), 32 deletions(-) create mode 100644 dlm_controld/node_config.c create mode 100644 dlm_controld/node_config.h
Signature · 1 line
2.26.2
PATCH v3 1/4 dlm_controld: add support for unsigned int values
This patch adds support for setting a unsigned integer value.
--- dlm_controld/config.c | 25 ++++++++++++++++ dlm_controld/dlm_daemon.h | 6 ++++ dlm_controld/main.c | 60 ++++++++++++++++++++++----------------- 3 files changed, 65 insertions(+), 26 deletions(-)
dlm_controld/config.c +25 -0
@@ -156,6 +156,19 @@ static void get_val_int(char *line, int *val_out) *val_out =atoi (val); } +static void get_val_uint (char *line,unsigned int *val_out) +{ +char key[MAX_LINE ]; +char val[MAX_LINE ]; +int rv; + + rv =sscanf (line,"%[^=]=%s" , key, val); +if (rv !=2 ) +return ; + + *val_out =strtoul (val,NULL ,0 ); +} +static void get_val_str (char *line,char *val_out) {char key[MAX_LINE ];
@@ -171,6 +184,7 @@ static void get_val_str(char *line, char *val_out)void set_opt_file (int update) { +unsigned int uval =0 ;struct dlm_option *o;FILE *file;char line[MAX_LINE ];
@@ -238,6 +252,17 @@ void set_opt_file(int update)log_debug ("config file %s = %d cli_set %d use %d" , o->name, o->file_int, o->cli_set, o->use_int); + }else if (o->req_arg == req_arg_uint) { +get_val_uint (line, &uval); + + o->file_uint = uval; + +if (!o->cli_set) + o->use_uint = o->file_uint; + +log_debug ("config file %s = %u cli_set %d use %u" , + o->name, o->file_uint, o->cli_set, o->use_uint); + }else if (o->req_arg == req_arg_bool) {get_val_int (line, &val);
dlm_controld/dlm_daemon.h +6 -0
@@ -86,6 +86,7 @@ enum { req_arg_bool =1 , req_arg_int =2 , req_arg_str =3 , + req_arg_uint = 4, }; enum {
@@ -125,22 +126,27 @@ struct dlm_option {int use_int;char *use_str; +unsigned int use_uint;int default_int;const char *default_str; +unsigned int default_uint;int cli_set;int cli_int;char *cli_str; +unsigned int cli_uint;int file_set;int file_int;char *file_str; +unsigned int file_uint; }; EXTERN struct dlm_option dlm_options[dlm_options_max];#define opt (x) dlm_options[x].use_int#define opts (x) dlm_options[x].use_str +#define optu (x) dlm_options[x].use_uint/* DLM_LOCKSPACE_LEN: maximum lockspace name length, from linux/dlmconstants.h.
dlm_controld/main.c +34 -26
188 lines not on this page · raw message
Signature · 1 line
2.26.2
PATCH v3 2/4 dlm_controld: set listen skb mark setting
In reply to Alexander AringThis patch adds support to set the skb mark value for the in-kernel DLM listen socket.
--- dlm_controld/action.c | 2 ++ dlm_controld/dlm.conf.5 | 2 ++ dlm_controld/dlm_daemon.h | 1 + dlm_controld/main.c | 5 +++++ 4 files changed, 10 insertions(+)
dlm_controld/action.c +2 -0
@@ -851,6 +851,8 @@ int setup_configfs_options(void) dlm_options[timewarn_ind].file_set)set_configfs_cluster ("timewarn_cs" ,NULL ,opt (timewarn_ind)); +set_configfs_cluster ("mark" ,NULL ,optu (mark_ind)); + proto_name =opts (protocol_ind); proto_num =-1 ;
dlm_controld/dlm.conf.5 +2 -0
@@ -40,6 +40,8 @@ protocol .br bind_all .br +mark +.br debug_logfile .br enable_plock
dlm_controld/dlm_daemon.h +1 -0
@@ -97,6 +97,7 @@ enum { protocol_ind, debug_logfile_ind, bind_all_ind, + mark_ind, enable_fscontrol_ind, enable_plock_ind, plock_debug_ind,
dlm_controld/main.c +5 -0
@@ -1737,6 +1737,11 @@ static void set_opt_defaults(void)0 ,NULL ,0 , ""); /* do not advertise */ +set_opt_default (mark_ind, +"mark" ,'\0' , req_arg_uint, +0 ,NULL ,0 , +"set mark value for the DLM in-kernel listen socket" ); + set_opt_default(debug_logfile_ind, "debug_logfile", 'L', no_arg, 0, NULL, 0,
Signature · 1 line
2.26.2
PATCH v3 3/4 dlm_controld: add support for per nodeid configuration
In reply to Alexander AringThis patch adds support to make a configuration per nodeid and key-value pairs. As example this patch will introduce the key mark to set via configfs comms per nodeid the SO_MARK socket option.
--- dlm_controld/Makefile | 3 +- dlm_controld/action.c | 36 +++++++++++++++-- dlm_controld/dlm.conf.5 | 19 +++++++++ dlm_controld/dlm_daemon.h | 5 ++- dlm_controld/main.c | 4 ++ dlm_controld/member.c | 6 ++- dlm_controld/node_config.c | 82 ++++++++++++++++++++++++++++++++++++++ dlm_controld/node_config.h | 31 ++++++++++++++ 8 files changed, 180 insertions(+), 6 deletions(-) create mode 100644 dlm_controld/node_config.c create mode 100644 dlm_controld/node_config.h
dlm_controld/Makefile +2 -1
@@ -32,7 +32,8 @@ BIN_SOURCE = action.c \ config.c \ member.c \ logging.c \ - rbtree.c + rbtree.c \ + node_config.c LIB_SOURCE = lib.c CFLAGS += -D_GNU_SOURCE -O2 -ggdb \
dlm_controld/action.c +33 -3
@@ -570,15 +570,16 @@ static int add_configfs_base(void)return rv; } -int add_configfs_node (int nodeid,char *addr,int addrlen,int local) +int add_configfs_node (int nodeid,char *addr,int addrlen,int local, +uint32_t mark) {char path[PATH_MAX ];char padded_addr[sizeof (struct sockaddr_storage )];char buf[32 ];int rv, fd; -log_debug ("set_configfs_node %d %s local %d" , - nodeid,str_ip (addr), local); +log_debug ("set_configfs_node %d %s local %d mark %" PRIu32, + nodeid,str_ip (addr), local, mark);/* * create comm dir for this node
@@ -639,6 +640,35 @@ int add_configfs_node(int nodeid, char *addr, int addrlen, int local) }close (fd); +/* +* set skb mark for nodeid +* +* If open() fails we skip it because kernel doesn't support it. +* It's not a required confiuration. It will show up in the log. +*/ + +memset (path,0 ,PATH_MAX ); +snprintf (path,PATH_MAX ,"%s/%d/mark" ,COMMS_DIR , nodeid); + + fd =open (path,O_WRONLY ); +if (fd <0 ) { +log_error ("%s: open failed: %d" , path, errno); + goto skip_non_required; + } + +memset (buf,0 ,sizeof (buf)); +snprintf (buf,32 ,"%" PRIu32, mark); + + rv =do_write (fd, buf,strlen (buf)); +if (rv <0 ) { +log_error ("%s: write failed: %d, %s" , path, errno, buf); +close (fd); +return -1 ; + } +close (fd); + +skip_non_required: +/* * set local */
dlm_controld/dlm.conf.5 +19 -0
@@ -392,6 +392,25 @@ master foo node=2 weight=1 In which case node 1 will master 2/3 of the total resources and node 2 will master the other 1/3. +.SS Node configuration + +Node configurations can be set by the node keyword followed of key-value +pairs. + +.B Keys: + +.B mark +The mark key can be used to set a specific mark value which is then used +by the in-kernel DLM socket creation. This can be used to match for DLM +specfic packets for e.g. routing. + +Example of setting a per socket value for nodeid 1 and a mark value +of 42: + +node id=1 mark=42 + +For local nodes this value doesn't have any effect. + .SH SEE ALSO .BR dlm_controld (8), .BR dlm_tool (8)
dlm_controld/dlm_daemon.h +4 -1
@@ -40,6 +40,7 @@#include <sched.h> #include <signal.h> #include <dirent.h> +#include <inttypes.h> #include <sys/sysmacros.h> #include <corosync/cpg.h>
@@ -48,6 +49,7 @@#include "libdlmcontrol.h" #include "dlm_controld.h" #include "fence_config.h" +#include "node_config.h" #include "list.h" #include "rbtree.h" #include "linux_endian.h"
@@ -365,7 +367,8 @@ int set_sysfs_nodir(char *name, int val);int set_configfs_members (struct lockspace *ls,char *name,int new_count,int *new_members,int renew_count,int *renew_members); -int add_configfs_node (int nodeid,char *addr,int addrlen,int local); +int add_configfs_node (int nodeid,char *addr,int addrlen,int local, +uint32_t mark);void del_configfs_node (int nodeid);void clear_configfs (void );int setup_configfs_options (void );
dlm_controld/main.c +4 -0
@@ -2052,6 +2052,10 @@ int main(int argc, char **argv)set_opt_cli (argc, argv);set_opt_file (0 ); + rv =node_config_init (CONF_FILE_PATH ); +if (rv) +return 1 ; +strcpy (fence_all_device.name,"fence_all" );strcpy (fence_all_device.agent,"dlm_stonith" ); fence_all_device.unfence =0 ;
dlm_controld/member.c +5 -1
@@ -109,6 +109,7 @@ static void quorum_callback(quorum_handle_t h, uint32_t quorate, {corosync_cfg_node_address_t addrs[MAX_NODE_ADDRESSES ];corosync_cfg_node_address_t *addrptr = addrs; +const struct node_config *nc;cs_error_t err;int i, j, num_addrs;uint64_t now =monotime ();
@@ -163,12 +164,15 @@ static void quorum_callback(quorum_handle_t h, uint32_t quorate,continue ; } + nc =node_config_get (quorum_nodes[i]); +for (j =0 ; j < num_addrs; j++) {add_configfs_node (quorum_nodes[i], addrptr[j].address, addrptr[j].address_length, (quorum_nodes[i] == - our_nodeid)); + our_nodeid), + nc->mark); } } }
dlm_controld/node_config.c new file +82 -0
82 lines not on this page · raw message
dlm_controld/node_config.h new file +31 -0
@@ -0,0 +1,31 @@ +/* +* Copyright 2020 Red Hat, Inc. +* +* This copyrighted material is made available to anyone wishing to use, +* modify, copy, or redistribute it subject to the terms and conditions +* of the GNU General Public License v2 or (at your option) any later version. +*/ + +#ifndef _NODE_CONFIG_H_ +#define _NODE_CONFIG_H_ + +#include <stdint.h> + +struct node_config { +uint32_t mark; +}; + +/* +* Returns -ENOENT if path does not exist or there is no +* config for nodeid in the file. +* +* Returns -EXYZ if there's a problem with the config. +* +* Returns 0 if a config was found with no problems. +*/ + +int node_config_init (const char *path); + +const struct node_config *node_config_get (int nodeid); + +#endif
Signature · 1 line
2.26.2
PATCH v3 4/4 dlm_controld: add support for waitplock_recovery switch
In reply to Alexander AringThis patch adds support to set the cluster attribute waitplock_recovery via enable_waitplock_recover arg or config file attribute.
--- dlm_controld/action.c | 5 +++++ dlm_controld/dlm.conf.5 | 2 ++ dlm_controld/dlm_daemon.h | 1 + dlm_controld/main.c | 5 +++++ 4 files changed, 13 insertions(+)
dlm_controld/action.c +5 -0
@@ -881,6 +881,11 @@ int setup_configfs_options(void) dlm_options[timewarn_ind].file_set)set_configfs_cluster ("timewarn_cs" ,NULL ,opt (timewarn_ind)); +if (dlm_options[enable_waitplock_recovery_ind].cli_set || + dlm_options[enable_waitplock_recovery_ind].file_set) +set_configfs_cluster ("waitplock_recovery" ,NULL , +opt (enable_waitplock_recovery_ind)); +set_configfs_cluster ("mark" ,NULL ,optu (mark_ind)); proto_name =opts (protocol_ind);
dlm_controld/dlm.conf.5 +2 -0
@@ -46,6 +46,8 @@ debug_logfile .br enable_plock .br +enable_waitplock_recovery +.br plock_debug .br plock_rate_limit
dlm_controld/dlm_daemon.h +1 -0
@@ -102,6 +102,7 @@ enum { mark_ind, enable_fscontrol_ind, enable_plock_ind, + enable_waitplock_recovery_ind, plock_debug_ind, plock_rate_limit_ind, plock_ownership_ind,
dlm_controld/main.c +5 -0
@@ -1757,6 +1757,11 @@ static void set_opt_defaults(void)1 ,NULL ,0 , "enable/disable posix lock support for cluster fs"); +set_opt_default (enable_waitplock_recovery_ind, +"enable_waitplock_recovery" ,'\0' , req_arg_bool, +0 ,NULL ,0 , +"enable/disable posix lock to wait for dlm recovery after lock acquire" ); + set_opt_default(plock_debug_ind, "plock_debug", 'P', no_arg, 0, NULL, 0,
Signature · 1 line
2.26.2