OCFS Configurable timeouts - Revision 5
PATCH 1/3 ocfs2-expose-o2nm_cluster.patch
In reply to [email protected]From: Andrew Beekhof <[email protected]>
Subject: [patch 1/3] OCFS2 - Expose struct o2nm_cluster
Subsequent patches (namely userspace heartbeat and configurable timeouts) require access to the o2nm_cluster struct. This patch does the necessary shuffling.
--- fs/ocfs2/cluster/nodemanager.c | 13 +------------ fs/ocfs2/cluster/nodemanager.h | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 12 deletions(-)
ocfs2/cluster/nodemanager.c renamed from ocfs2/cluster/nodemanager.c.orig +1 -12
@@ -35,7 +35,7 @@/* for now we operate under the assertion that there can be only one * cluster active at a time. Changing this will require trickling * cluster references throughout where nodes are looked up */ -static struct o2nm_cluster *o2nm_single_cluster =NULL ; +struct o2nm_cluster *o2nm_single_cluster =NULL ;#define OCFS2_MAX_HB_CTL_PATH 256static char ocfs2_hb_ctl_path[OCFS2_MAX_HB_CTL_PATH ] ="/sbin/ocfs2_hb_ctl" ;
@@ -97,17 +97,6 @@ const char *o2nm_get_hb_ctl_path(void) }EXPORT_SYMBOL_GPL (o2nm_get_hb_ctl_path); -struct o2nm_cluster { -struct config_group cl_group; -unsigned cl_has_local:1 ; -u8 cl_local_node; -rwlock_t cl_nodes_lock; -struct o2nm_node *cl_nodes[O2NM_MAX_NODES ]; -struct rb_root cl_node_ip_tree; -/* this bitmap is part of a hack for disk bitmap.. will go eventually. - zab */ -unsigned long cl_nodes_bitmap[BITS_TO_LONGS (O2NM_MAX_NODES )]; -}; -struct o2nm_node *o2nm_get_node_by_num (u8 node_num) {struct o2nm_node *node =NULL ;
ocfs2/cluster/nodemanager.h renamed from ocfs2/cluster/nodemanager.h.orig +14 -0
@@ -53,6 +53,20 @@ struct o2nm_node {unsigned long nd_set_attributes; }; +struct o2nm_cluster { +struct config_group cl_group; +unsigned cl_has_local:1 ; +u8 cl_local_node; +rwlock_t cl_nodes_lock; +struct o2nm_node *cl_nodes[O2NM_MAX_NODES ]; +struct rb_root cl_node_ip_tree; + +/* this bitmap is part of a hack for disk bitmap.. will go eventually. - zab */ +unsigned long cl_nodes_bitmap[BITS_TO_LONGS (O2NM_MAX_NODES )]; +}; + +extern struct o2nm_cluster *o2nm_single_cluster; +u8 o2nm_this_node (void );int o2nm_configured_node_map (unsigned long *map,unsigned bytes);
--
cover OCFS Configurable timeouts - Revision 5
Minor improvements to the handshake logic
- Ensure failed handshakes are only processed once
- Skip the o2net_msg code after a sucessful handshake
Do we need logic to handle inbound messages that arrive before a valid handshake? Or does this not happen?
--
Andrew
PATCH 2/3 ocfs2-configurable-timeout.patch
From: Jeff Mahoney <[email protected]>
Subject: [patch 2/3] OCFS2 Configurable timeouts
Allow configuration of OCFS2 timeouts from userspace via configfs
--- fs/ocfs2/cluster/nodemanager.c | 161 ++++++++++++++++++++++++++++++++++++++++ fs/ocfs2/cluster/nodemanager.h | 3 fs/ocfs2/cluster/tcp.c | 60 +++++++++++--- fs/ocfs2/cluster/tcp.h | 7 + fs/ocfs2/cluster/tcp_internal.h | 6 - 5 files changed, 219 insertions(+), 18 deletions(-)
ocfs2/cluster/nodemanager.c renamed from ocfs2/cluster/nodemanager.c.orig +161 -0
@@ -532,6 +532,161 @@ static struct o2nm_node_group *to_o2nm_n }#endif +struct o2nm_cluster_attribute { +struct configfs_attribute attr; +ssize_t (*show)(struct o2nm_cluster *,char *); +ssize_t (*store)(struct o2nm_cluster *,const char *,size_t ); +}; + +static ssize_t o2nm_cluster_attr_write (const char *page,ssize_t count, +unsigned int *val) +{ +unsigned long tmp; +char *p = (char *)page; + + tmp =simple_strtoul (p, &p,0 ); +if (!p || (*p && (*p !='\n' ))) +return -EINVAL ; + +if (tmp ==0 ) +return -EINVAL ; +if (tmp >= (u32)-1 ) +return -ERANGE ; + + *val = tmp; + +return count; +} + +static ssize_t o2nm_cluster_attr_idle_timeout_ms_read ( +struct o2nm_cluster *cluster,char *page) +{ +return sprintf (page,"%u\n" , cluster->cl_idle_timeout_ms); +} + +static ssize_t o2nm_cluster_attr_idle_timeout_ms_write ( +struct o2nm_cluster *cluster,const char *page,size_t count) +{ +ssize_t ret; +unsigned int val; + + ret =o2nm_cluster_attr_write (page, count, &val); + +if (ret >0 ) { +if (val <= cluster->cl_keepalive_delay_ms) { +mlog (ML_NOTICE ,"o2net: idle timeout must be larger " +"than keepalive delay\n" ); +return -EINVAL ; + } + cluster->cl_idle_timeout_ms = val; + } + +return ret; +} + +static ssize_t o2nm_cluster_attr_keepalive_delay_ms_read ( +struct o2nm_cluster *cluster,char *page) +{ +return sprintf (page,"%u\n" , cluster->cl_keepalive_delay_ms); +} + +static ssize_t o2nm_cluster_attr_keepalive_delay_ms_write ( +struct o2nm_cluster *cluster,const char *page,size_t count) +{ +ssize_t ret; +unsigned int val; + + ret =o2nm_cluster_attr_write (page, count, &val); + +if (ret >0 ) { +if (val >= cluster->cl_idle_timeout_ms) { +mlog (ML_NOTICE ,"o2net: keepalive delay must be " +"smaller than idle timeout\n" ); +return -EINVAL ; + } + cluster->cl_keepalive_delay_ms = val; + } + +return ret; +} + +static ssize_t o2nm_cluster_attr_reconnect_delay_ms_read ( +struct o2nm_cluster *cluster,char *page) +{ +return sprintf (page,"%u\n" , cluster->cl_reconnect_delay_ms); +} + +static ssize_t o2nm_cluster_attr_reconnect_delay_ms_write ( +struct o2nm_cluster *cluster,const char *page,size_t count) +{ +return o2nm_cluster_attr_write (page, count, + &cluster->cl_reconnect_delay_ms); +} +static struct o2nm_cluster_attribute o2nm_cluster_attr_idle_timeout_ms = { + .attr = { .ca_owner =THIS_MODULE , + .ca_name ="idle_timeout_ms" , + .ca_mode =S_IRUGO |S_IWUSR }, + .show = o2nm_cluster_attr_idle_timeout_ms_read, + .store = o2nm_cluster_attr_idle_timeout_ms_write, +}; + +static struct o2nm_cluster_attribute o2nm_cluster_attr_keepalive_delay_ms = { + .attr = { .ca_owner =THIS_MODULE , + .ca_name ="keepalive_delay_ms" , + .ca_mode =S_IRUGO |S_IWUSR }, + .show = o2nm_cluster_attr_keepalive_delay_ms_read, + .store = o2nm_cluster_attr_keepalive_delay_ms_write, +}; + +static struct o2nm_cluster_attribute o2nm_cluster_attr_reconnect_delay_ms = { + .attr = { .ca_owner =THIS_MODULE , + .ca_name ="reconnect_delay_ms" , + .ca_mode =S_IRUGO |S_IWUSR }, + .show = o2nm_cluster_attr_reconnect_delay_ms_read, + .store = o2nm_cluster_attr_reconnect_delay_ms_write, +}; + +static struct configfs_attribute *o2nm_cluster_attrs[] = { + &o2nm_cluster_attr_idle_timeout_ms.attr, + &o2nm_cluster_attr_keepalive_delay_ms.attr, + &o2nm_cluster_attr_reconnect_delay_ms.attr, +NULL , +}; +static ssize_t o2nm_cluster_show (struct config_item *item, +struct configfs_attribute *attr, +char *page) +{ +struct o2nm_cluster *cluster =to_o2nm_cluster (item); +struct o2nm_cluster_attribute *o2nm_cluster_attr = + container_of(attr, struct o2nm_cluster_attribute, attr); +ssize_t ret =0 ; + +if (o2nm_cluster_attr->show) + ret = o2nm_cluster_attr->show (cluster, page); +return ret; +} + +static ssize_t o2nm_cluster_store (struct config_item *item, +struct configfs_attribute *attr, +const char *page,size_t count) +{ +struct o2nm_cluster *cluster =to_o2nm_cluster (item); +struct o2nm_cluster_attribute *o2nm_cluster_attr = + container_of(attr, struct o2nm_cluster_attribute, attr); +ssize_t ret; + +if (o2nm_cluster_attr->store ==NULL ) { + ret = -EINVAL ; + goto out; + } + + ret = o2nm_cluster_attr->store (cluster, page, count); +if (ret < count) + goto out; +out: +return ret; +} +static struct config_item *o2nm_node_group_make_item (struct config_group *group,const char *name) {
@@ -613,10 +768,13 @@ static void o2nm_cluster_release(structstatic struct configfs_item_operations o2nm_cluster_item_ops = { .release = o2nm_cluster_release, + .show_attribute = o2nm_cluster_show, + .store_attribute = o2nm_cluster_store, };static struct config_item_type o2nm_cluster_type = { .ct_item_ops = &o2nm_cluster_item_ops, + .ct_attrs = o2nm_cluster_attrs, .ct_owner =THIS_MODULE , };
@@ -667,6 +825,9 @@ static struct config_group *o2nm_cluster cluster->cl_group.default_groups[2 ] =NULL ;rwlock_init (&cluster->cl_nodes_lock); cluster->cl_node_ip_tree =RB_ROOT ; + cluster->cl_reconnect_delay_ms =O2NET_RECONNECT_DELAY_MS_DEFAULT ; + cluster->cl_idle_timeout_ms =O2NET_IDLE_TIMEOUT_MS_DEFAULT ; + cluster->cl_keepalive_delay_ms =O2NET_KEEPALIVE_DELAY_MS_DEFAULT ; ret = &cluster->cl_group; o2nm_single_cluster = cluster;
ocfs2/cluster/nodemanager.h renamed from ocfs2/cluster/nodemanager.h.orig +3 -0
@@ -60,6 +60,9 @@ struct o2nm_cluster {rwlock_t cl_nodes_lock;struct o2nm_node *cl_nodes[O2NM_MAX_NODES ];struct rb_root cl_node_ip_tree; +unsigned int cl_idle_timeout_ms; +unsigned int cl_keepalive_delay_ms; +unsigned int cl_reconnect_delay_ms;/* this bitmap is part of a hack for disk bitmap.. will go eventually. - zab */ unsigned long cl_nodes_bitmap[BITS_TO_LONGS (O2NM_MAX_NODES )];
ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +48 -12
121 lines not on this page · raw message
ocfs2/cluster/tcp.h renamed from ocfs2/cluster/tcp.h.orig +7 -0
@@ -54,6 +54,13 @@ typedef int (o2net_msg_handler_func)(str#define O2NET_MAX_PAYLOAD_BYTES (4096 - sizeof(struct o2net_msg)) +/* same as hb delay, we're waiting for another node to recognize our hb */ +#define O2NET_RECONNECT_DELAY_MS_DEFAULT 2000 + +#define O2NET_KEEPALIVE_DELAY_MS_DEFAULT 5000 +#define O2NET_IDLE_TIMEOUT_MS_DEFAULT 10000 + +/* TODO: figure this out.... */ static inline int o2net_link_down (int err,struct socket *sock) {
ocfs2/cluster/tcp_internal.h renamed from ocfs2/cluster/tcp_internal.h.orig +0 -6
@@ -27,17 +27,11 @@#define O2NET_MSG_KEEP_REQ_MAGIC ((u16)0xfa57)#define O2NET_MSG_KEEP_RESP_MAGIC ((u16)0xfa58) -/* same as hb delay, we're waiting for another node to recognize our hb */ -#define O2NET_RECONNECT_DELAY_MS O2HB_REGION_TIMEOUT_MS -/* we're delaying our quorum decision so that heartbeat will have timed * out truly dead nodes by the time we come around to making decisions * on their number */ #define O2NET_QUORUM_DELAY_MS ((o2hb_dead_threshold + 2) * O2HB_REGION_TIMEOUT_MS) -#define O2NET_KEEPALIVE_DELAY_SECS 5 -#define O2NET_IDLE_TIMEOUT_SECS 10 -/* * This version number represents quite a lot, unfortunately. It not * only represents the raw network message protocol on the wire but also
--
PATCH 3/3 ocfs2-timeout-protocol.patch
In reply to [email protected]From: Andrew Beekhof <[email protected]>
Subject: [patch 3/3] OCFS2 Configurable timeouts - Protocol changes
Modify the OCFS2 handshake to ensure essential timeouts are configured identically on all nodes.
Only allow changes when there are no connected peers
Improves the logic in o2net_advance_rx() which broke now that sizeof(struct o2net_handshake) is greater than sizeof(struct o2net_msg)
Included is the field for userspace-heartbeat timeout to avoid the need for further protocol changes.
Uses a global spinlock to ensure the decisions to update configfs entries are made on the correct value. The region covered by the spinlock when incrimenting the counter is much larger as this is the more critical case.
--- fs/ocfs2/cluster/nodemanager.c | 19 +++++++ fs/ocfs2/cluster/tcp.c | 96 +++++++++++++++++++++++++++++++++++----- fs/ocfs2/cluster/tcp.h | 1 fs/ocfs2/cluster/nodemanager.c | 30 ++++++++++-- fs/ocfs2/cluster/tcp.c | 94 ++++++++++++++++++++++++++++++++++++---- fs/ocfs2/cluster/tcp.h | 1 fs/ocfs2/cluster/tcp_internal.h | 6 ++ 4 files changed, 115 insertions(+), 16 deletions(-)
ocfs2/cluster/nodemanager.c renamed from ocfs2/cluster/nodemanager.c.orig +24 -6
@@ -573,12 +573,21 @@ static ssize_t o2nm_cluster_attr_idle_ti ret =o2nm_cluster_attr_write (page, count, &val);if (ret >0 ) { -if (val <= cluster->cl_keepalive_delay_ms) { +if (cluster->cl_idle_timeout_ms != val + &&o2net_num_connected_peers ()) { +mlog (ML_NOTICE , +"o2net: cannot change idle timeout after " +"the first peer has agreed to it." +" %d connected peers\n" , +o2net_num_connected_peers ()); + ret = -EINVAL ; + }else if (val <= cluster->cl_keepalive_delay_ms) {mlog (ML_NOTICE ,"o2net: idle timeout must be larger " "than keepalive delay\n" ); -return -EINVAL ; + ret = -EINVAL ; + }else { + cluster->cl_idle_timeout_ms = val; } - cluster->cl_idle_timeout_ms = val; }return ret;
@@ -599,12 +608,21 @@ static ssize_t o2nm_cluster_attr_keepali ret =o2nm_cluster_attr_write (page, count, &val);if (ret >0 ) { -if (val >= cluster->cl_idle_timeout_ms) { +if (cluster->cl_keepalive_delay_ms != val + &&o2net_num_connected_peers ()) { +mlog (ML_NOTICE , +"o2net: cannot change keepalive delay after" +" the first peer has agreed to it." +" %d connected peers\n" , +o2net_num_connected_peers ()); + ret = -EINVAL ; + }else if (val >= cluster->cl_idle_timeout_ms) {mlog (ML_NOTICE ,"o2net: keepalive delay must be " "smaller than idle timeout\n" ); -return -EINVAL ; + ret = -EINVAL ; + }else { + cluster->cl_keepalive_delay_ms = val; } - cluster->cl_keepalive_delay_ms = val; }return ret;
ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +85 -9
148 lines not on this page · raw message
ocfs2/cluster/tcp.h renamed from ocfs2/cluster/tcp.h.orig +1 -0
@@ -108,6 +108,7 @@ void o2net_unregister_hb_callbacks(void)int o2net_start_listening (struct o2nm_node *node);void o2net_stop_listening (struct o2nm_node *node);void o2net_disconnect_node (struct o2nm_node *node); +int o2net_num_connected_peers (void );int o2net_init (void );void o2net_exit (void );
ocfs2/cluster/tcp_internal.h renamed from ocfs2/cluster/tcp_internal.h.orig +5 -1
@@ -48,10 +48,14 @@* - full 64 bit i_size in the metadata lock lvbs * - introduction of "rw" lock and pushing meta/data locking down */ -#define O2NET_PROTOCOL_VERSION 4ULL +#define O2NET_PROTOCOL_VERSION 5ULLstruct o2net_handshake {__be64 protocol_version;__be64 connector_id; +__be32 o2hb_heartbeat_timeout_ms; +__be32 o2net_idle_timeout_ms; +__be32 o2net_keepalive_delay_ms; +__be32 o2net_reconnect_delay_ms; };struct o2net_node {
--
PATCH 3/3 ocfs2-timeout-protocol.patch
ocfs2/cluster/tcp.c static void o2net_set_nn_state(struct o2 · line 405
Quoted 7 lines
ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +6
-0+if (old_sc != sc) { +if (old_sc) +atomic_dec (&o2net_connected_peers); +else +atomic_inc (&o2net_connected_peers); + }
It occurs to me that this probably doesn't catch all cases. Imagine old_sc and sc both being non-null. It'd dec when it shouldn't change.
How about:
if (old_sc && !sc)
dec
else if (!old_sc && sc)
inc
Otherwise it seems OK.
- z
PATCH 3/3 ocfs2-timeout-protocol.patch
Quoted 2 lines · Zach Brown
It occurs to me that this probably doesn't catch all cases. Imagine
old_scand sc both being non-null. It'd dec when it shouldn't change.
I made the change myself and merged the patch series into ocfs2.git.
Thanks, Andrew.
--Mark
--
Mark Fasheh
Senior Software Developer, Oracle
[email protected]