dlm_controld: support for mark and waitplock_recovery
v2 cover dlm_controld: support for mark and waitplock_recovery
Hi,
this patch series adds support for set the in-kernel socket skb mark value over dlm_controld. There exists two kinds of socket, one listen socket and multiple peer sockets. Both can be set via the dlm config file via "listen_mark" or multiple entries of:
node id=$NODEID mark=$MARK
whereas $NODEID is the corosync assigned nodeid. The given mark number can be hexadecimal or decimal.
Also it adds support to set the waitplock_recovery per cluster attribute by setting enable_waitplock_recover over file or argument configuration.
- Alex
changes since v2:
- remove leftover PRIu32 in nodeid configuration
- make unsigned int values also work with file configuration, was arg only before
- add support to set the waitplock_recovery switch via dlm_controld
- remove free function of node_config, may be necessary when implement some kind of NOHUP and reparse config file
Alexander Aring (4):
dlm_controld: add support for unsigned int values
dlm_controld: set listen skb mark setting
dlm_controld: add support for per nodeid configuration
dlm_controld: add support for waitplock_recovery switch
dlm_controld/Makefile | 3 +- dlm_controld/action.c | 38 ++++++++++++++++-- dlm_controld/config.c | 25 ++++++++++++ dlm_controld/dlm.conf.5 | 23 +++++++++++ dlm_controld/dlm_daemon.h | 12 +++++- dlm_controld/main.c | 17 ++++++++ dlm_controld/member.c | 6 ++- dlm_controld/node_config.c | 82 ++++++++++++++++++++++++++++++++++++++ dlm_controld/node_config.h | 31 ++++++++++++++ 9 files changed, 231 insertions(+), 6 deletions(-) create mode 100644 dlm_controld/node_config.c create mode 100644 dlm_controld/node_config.h
Signature · 1 line
2.26.2
PATCH v2 1/4 dlm_controld: add support for unsigned int values
This patch adds support for setting a unsigned integer value.
--- dlm_controld/config.c | 25 +++++++++++++++++++++++++ dlm_controld/dlm_daemon.h | 5 +++++ dlm_controld/main.c | 3 +++ 3 files changed, 33 insertions(+)
dlm_controld/config.c +25 -0
@@ -156,6 +156,19 @@ static void get_val_int(char *line, int *val_out) *val_out =atoi (val); } +static void get_val_uint (char *line,unsigned int *val_out) +{ +char key[MAX_LINE ]; +char val[MAX_LINE ]; +int rv; + + rv =sscanf (line,"%[^=]=%s" , key, val); +if (rv !=2 ) +return ; + + *val_out =strtoul (val,NULL ,0 ); +} +static void get_val_str (char *line,char *val_out) {char key[MAX_LINE ];
@@ -171,6 +184,7 @@ static void get_val_str(char *line, char *val_out)void set_opt_file (int update) { +unsigned int uval =0 ;struct dlm_option *o;FILE *file;char line[MAX_LINE ];
@@ -238,6 +252,17 @@ void set_opt_file(int update)log_debug ("config file %s = %d cli_set %d use %d" , o->name, o->file_int, o->cli_set, o->use_int); + }else if (o->req_arg == req_arg_uint) { +get_val_uint (line, &uval); + + o->file_uint = uval; + +if (!o->cli_set) + o->use_uint = o->file_uint; + +log_debug ("config file %s = %u cli_set %d use %u" , + o->name, o->file_uint, o->cli_set, o->use_uint); + }else if (o->req_arg == req_arg_bool) {get_val_int (line, &val);
dlm_controld/dlm_daemon.h +5 -0
@@ -86,6 +86,7 @@ enum { req_arg_bool =1 , req_arg_int =2 , req_arg_str =3 , + req_arg_uint = 4, }; enum {
@@ -125,6 +126,7 @@ struct dlm_option {int use_int;char *use_str; +unsigned int use_uint;int default_int;const char *default_str;
@@ -132,15 +134,18 @@ struct dlm_option {int cli_set;int cli_int;char *cli_str; +unsigned int cli_uint;int file_set;int file_int;char *file_str; +unsigned int file_uint; }; EXTERN struct dlm_option dlm_options[dlm_options_max];#define opt (x) dlm_options[x].use_int#define opts (x) dlm_options[x].use_str +#define optu (x) dlm_options[x].use_uint/* DLM_LOCKSPACE_LEN: maximum lockspace name length, from linux/dlmconstants.h.
dlm_controld/main.c +3 -0
@@ -1972,6 +1972,9 @@ static void set_opt_cli(int argc, char **argv) }else if (o->req_arg == req_arg_bool) { o->cli_int =atoi (arg_str) ?1 :0 ; o->use_int = o->cli_int; + }else if (o->req_arg == req_arg_uint) { + o->cli_uint =strtoul (arg_str,NULL ,0 ); + o->use_uint = o->cli_uint; } }
Signature · 1 line
2.26.2
PATCH v2 2/4 dlm_controld: set listen skb mark setting
In reply to Alexander AringThis patch adds support to set the skb mark value for the in-kernel DLM listen socket.
--- dlm_controld/action.c | 2 ++ dlm_controld/dlm.conf.5 | 2 ++ dlm_controld/dlm_daemon.h | 1 + dlm_controld/main.c | 5 +++++ 4 files changed, 10 insertions(+)
dlm_controld/action.c +2 -0
@@ -851,6 +851,8 @@ int setup_configfs_options(void) dlm_options[timewarn_ind].file_set)set_configfs_cluster ("timewarn_cs" ,NULL ,opt (timewarn_ind)); +set_configfs_cluster ("mark" ,NULL ,optu (mark_ind)); + proto_name =opts (protocol_ind); proto_num =-1 ;
dlm_controld/dlm.conf.5 +2 -0
@@ -40,6 +40,8 @@ protocol .br bind_all .br +mark +.br debug_logfile .br enable_plock
dlm_controld/dlm_daemon.h +1 -0
@@ -97,6 +97,7 @@ enum { protocol_ind, debug_logfile_ind, bind_all_ind, + mark_ind, enable_fscontrol_ind, enable_plock_ind, plock_debug_ind,
dlm_controld/main.c +5 -0
@@ -1732,6 +1732,11 @@ static void set_opt_defaults(void)0 ,NULL , ""); /* do not advertise */ +set_opt_default (mark_ind, +"mark" ,'\0' , req_arg_uint, +0 ,NULL , +"set mark value for the DLM in-kernel listen socket" ); + set_opt_default(debug_logfile_ind, "debug_logfile", 'L', no_arg, 0, NULL,
Signature · 1 line
2.26.2
PATCH v2 3/4 dlm_controld: add support for per nodeid configuration
In reply to Alexander AringThis patch adds support to make a configuration per nodeid and key-value pairs. As example this patch will introduce the key mark to set via configfs comms per nodeid the SO_MARK socket option.
--- dlm_controld/Makefile | 3 +- dlm_controld/action.c | 31 ++++++++++++-- dlm_controld/dlm.conf.5 | 19 +++++++++ dlm_controld/dlm_daemon.h | 5 ++- dlm_controld/main.c | 4 ++ dlm_controld/member.c | 6 ++- dlm_controld/node_config.c | 82 ++++++++++++++++++++++++++++++++++++++ dlm_controld/node_config.h | 31 ++++++++++++++ 8 files changed, 175 insertions(+), 6 deletions(-) create mode 100644 dlm_controld/node_config.c create mode 100644 dlm_controld/node_config.h
dlm_controld/Makefile +2 -1
@@ -32,7 +32,8 @@ BIN_SOURCE = action.c \ config.c \ member.c \ logging.c \ - rbtree.c + rbtree.c \ + node_config.c LIB_SOURCE = lib.c CFLAGS += -D_GNU_SOURCE -O2 -ggdb \
dlm_controld/action.c +28 -3
@@ -570,15 +570,16 @@ static int add_configfs_base(void)return rv; } -int add_configfs_node (int nodeid,char *addr,int addrlen,int local) +int add_configfs_node (int nodeid,char *addr,int addrlen,int local, +uint32_t mark) {char path[PATH_MAX ];char padded_addr[sizeof (struct sockaddr_storage )];char buf[32 ];int rv, fd; -log_debug ("set_configfs_node %d %s local %d" , - nodeid,str_ip (addr), local); +log_debug ("set_configfs_node %d %s local %d mark %" PRIu32, + nodeid,str_ip (addr), local, mark);/* * create comm dir for this node
@@ -639,6 +640,30 @@ int add_configfs_node(int nodeid, char *addr, int addrlen, int local) }close (fd); +/* +* set skb mark for nodeid +*/ + +memset (path,0 ,PATH_MAX ); +snprintf (path,PATH_MAX ,"%s/%d/mark" ,COMMS_DIR , nodeid); + + fd =open (path,O_WRONLY ); +if (fd <0 ) { +log_error ("%s: open failed: %d" , path, errno); +return -1 ; + } + +memset (buf,0 ,sizeof (buf)); +snprintf (buf,32 ,"%" PRIu32, mark); + + rv =do_write (fd, buf,strlen (buf)); +if (rv <0 ) { +log_error ("%s: write failed: %d, %s" , path, errno, buf); +close (fd); +return -1 ; + } +close (fd); +/* * set local */
dlm_controld/dlm.conf.5 +19 -0
@@ -392,6 +392,25 @@ master foo node=2 weight=1 In which case node 1 will master 2/3 of the total resources and node 2 will master the other 1/3. +.SS Node configuration + +Node configurations can be set by the node keyword followed of key-value +pairs. + +.B Keys: + +.B mark +The mark key can be used to set a specific mark value which is then used +by the in-kernel DLM socket creation. This can be used to match for DLM +specfic packets for e.g. routing. + +Example of setting a per socket value for nodeid 1 and a mark value +of 42: + +node id=1 mark=42 + +For local nodes this value doesn't have any effect. + .SH SEE ALSO .BR dlm_controld (8), .BR dlm_tool (8)
dlm_controld/dlm_daemon.h +4 -1
@@ -40,6 +40,7 @@#include <sched.h> #include <signal.h> #include <dirent.h> +#include <inttypes.h> #include <sys/sysmacros.h> #include <corosync/cpg.h>
@@ -48,6 +49,7 @@#include "libdlmcontrol.h" #include "dlm_controld.h" #include "fence_config.h" +#include "node_config.h" #include "list.h" #include "rbtree.h" #include "linux_endian.h"
@@ -364,7 +366,8 @@ int set_sysfs_nodir(char *name, int val);int set_configfs_members (struct lockspace *ls,char *name,int new_count,int *new_members,int renew_count,int *renew_members); -int add_configfs_node (int nodeid,char *addr,int addrlen,int local); +int add_configfs_node (int nodeid,char *addr,int addrlen,int local, +uint32_t mark);void del_configfs_node (int nodeid);void clear_configfs (void );int setup_configfs_options (void );
dlm_controld/main.c +4 -0
@@ -2047,6 +2047,10 @@ int main(int argc, char **argv)set_opt_cli (argc, argv);set_opt_file (0 ); + rv =node_config_init (CONF_FILE_PATH ); +if (rv) +return 1 ; +strcpy (fence_all_device.name,"fence_all" );strcpy (fence_all_device.agent,"dlm_stonith" ); fence_all_device.unfence =0 ;
dlm_controld/member.c +5 -1
@@ -109,6 +109,7 @@ static void quorum_callback(quorum_handle_t h, uint32_t quorate, {corosync_cfg_node_address_t addrs[MAX_NODE_ADDRESSES ];corosync_cfg_node_address_t *addrptr = addrs; +const struct node_config *nc;cs_error_t err;int i, j, num_addrs;uint64_t now =monotime ();
@@ -163,12 +164,15 @@ static void quorum_callback(quorum_handle_t h, uint32_t quorate,continue ; } + nc =node_config_get (quorum_nodes[i]); +for (j =0 ; j < num_addrs; j++) {add_configfs_node (quorum_nodes[i], addrptr[j].address, addrptr[j].address_length, (quorum_nodes[i] == - our_nodeid)); + our_nodeid), + nc->mark); } } }
dlm_controld/node_config.c new file +82 -0
82 lines not on this page · raw message
dlm_controld/node_config.h new file +31 -0
@@ -0,0 +1,31 @@ +/* +* Copyright 2020 Red Hat, Inc. +* +* This copyrighted material is made available to anyone wishing to use, +* modify, copy, or redistribute it subject to the terms and conditions +* of the GNU General Public License v2 or (at your option) any later version. +*/ + +#ifndef _NODE_CONFIG_H_ +#define _NODE_CONFIG_H_ + +#include <stdint.h> + +struct node_config { +uint32_t mark; +}; + +/* +* Returns -ENOENT if path does not exist or there is no +* config for nodeid in the file. +* +* Returns -EXYZ if there's a problem with the config. +* +* Returns 0 if a config was found with no problems. +*/ + +int node_config_init (const char *path); + +const struct node_config *node_config_get (int nodeid); + +#endif
Signature · 1 line
2.26.2
PATCH v2 4/4 dlm_controld: add support for waitplock_recovery switch
In reply to Alexander AringThis patch adds support to set the cluster attribute waitplock_recovery via enable_waitplock_recover arg or config file attribute.
--- dlm_controld/action.c | 5 +++++ dlm_controld/dlm.conf.5 | 2 ++ dlm_controld/dlm_daemon.h | 1 + dlm_controld/main.c | 5 +++++ 4 files changed, 13 insertions(+)
dlm_controld/action.c +5 -0
@@ -876,6 +876,11 @@ int setup_configfs_options(void) dlm_options[timewarn_ind].file_set)set_configfs_cluster ("timewarn_cs" ,NULL ,opt (timewarn_ind)); +if (dlm_options[enable_waitplock_recovery_ind].cli_set || + dlm_options[enable_waitplock_recovery_ind].file_set) +set_configfs_cluster ("waitplock_recovery" ,NULL , +opt (enable_waitplock_recovery_ind)); +set_configfs_cluster ("mark" ,NULL ,optu (mark_ind)); proto_name =opts (protocol_ind);
dlm_controld/dlm.conf.5 +2 -0
@@ -46,6 +46,8 @@ debug_logfile .br enable_plock .br +enable_waitplock_recovery +.br plock_debug .br plock_rate_limit
dlm_controld/dlm_daemon.h +1 -0
@@ -102,6 +102,7 @@ enum { mark_ind, enable_fscontrol_ind, enable_plock_ind, + enable_waitplock_recovery_ind, plock_debug_ind, plock_rate_limit_ind, plock_ownership_ind,
dlm_controld/main.c +5 -0
@@ -1752,6 +1752,11 @@ static void set_opt_defaults(void)1 ,NULL , "enable/disable posix lock support for cluster fs"); +set_opt_default (enable_waitplock_recovery_ind, +"enable_waitplock_recovery" ,'\0' , req_arg_bool, +1 ,NULL , +"enable/disable posix lock to wait for dlm recovery after lock acquire" ); + set_opt_default(plock_debug_ind, "plock_debug", 'P', no_arg, 0, NULL,
Signature · 1 line
2.26.2
PATCH v2 4/4 dlm_controld: add support for waitplock_recovery switch
Hi,
Quoted 3 lines · Alexander Aring
This patch adds support to set the cluster attribute
waitplock_recoveryviaenable_waitplock_recoverarg or config file attribute.
...
dlm_controld/main.c static void set_opt_defaults(void) · line 1757
Quoted 11 lines
dlm_controld/main.c +3
-0@@ -1752,6 +1752,11 @@ static void set_opt_defaults(void)1 ,NULL , "enable/disable posix lock support for cluster fs"); + set_opt_default(enable_waitplock_recovery_ind, + "enable_waitplock_recovery", '\0', req_arg_bool, + 1, NULL,
that should be 0 otherwise the userland will activate it by default. The kernel default value is 0.
I will send a v3, sorry about that.
- Alex