OCFS Configurable timeouts from userspace
cover OCFS Configurable timeouts from userspace
--
Jeff has asked me to finish off his configurable timeouts patch.... here's the results :-)
One thing I'm not so happy about are the changes (I made) to o2net_msg struct in order to satisfy the logic in o2net_check_handshake()
Presumably the logic in o2net_check_handshake() should be changed?
Andrew
PATCH 1/3 OCFS2 - Expose struct o2nm_cluster
--- fs/ocfs2/cluster/nodemanager.c | 13 +------------ fs/ocfs2/cluster/nodemanager.h | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 12 deletions(-)
ocfs2/cluster/nodemanager.c renamed from ocfs2/cluster/nodemanager.c.orig +1 -12
@@ -35,7 +35,7 @@/* for now we operate under the assertion that there can be only one * cluster active at a time. Changing this will require trickling * cluster references throughout where nodes are looked up */ -static struct o2nm_cluster *o2nm_single_cluster =NULL ; +struct o2nm_cluster *o2nm_single_cluster =NULL ;#define OCFS2_MAX_HB_CTL_PATH 256static char ocfs2_hb_ctl_path[OCFS2_MAX_HB_CTL_PATH ] ="/sbin/ocfs2_hb_ctl" ;
@@ -97,17 +97,6 @@ const char *o2nm_get_hb_ctl_path(void) }EXPORT_SYMBOL_GPL (o2nm_get_hb_ctl_path); -struct o2nm_cluster { -struct config_group cl_group; -unsigned cl_has_local:1 ; -u8 cl_local_node; -rwlock_t cl_nodes_lock; -struct o2nm_node *cl_nodes[O2NM_MAX_NODES ]; -struct rb_root cl_node_ip_tree; -/* this bitmap is part of a hack for disk bitmap.. will go eventually. - zab */ -unsigned long cl_nodes_bitmap[BITS_TO_LONGS (O2NM_MAX_NODES )]; -}; -struct o2nm_node *o2nm_get_node_by_num (u8 node_num) {struct o2nm_node *node =NULL ;
ocfs2/cluster/nodemanager.h renamed from ocfs2/cluster/nodemanager.h.orig +14 -0
@@ -53,6 +53,20 @@ struct o2nm_node {unsigned long nd_set_attributes; }; +struct o2nm_cluster { +struct config_group cl_group; +unsigned cl_has_local:1 ; +u8 cl_local_node; +rwlock_t cl_nodes_lock; +struct o2nm_node *cl_nodes[O2NM_MAX_NODES ]; +struct rb_root cl_node_ip_tree; + +/* this bitmap is part of a hack for disk bitmap.. will go eventually. - zab */ +unsigned long cl_nodes_bitmap[BITS_TO_LONGS (O2NM_MAX_NODES )]; +}; + +extern struct o2nm_cluster *o2nm_single_cluster; +u8 o2nm_this_node (void );int o2nm_configured_node_map (unsigned long *map,unsigned bytes);
--
PATCH 3/3 OCFS2 Configurable timeouts - Protocol changes
In reply to [email protected]The addition of two dummy fields is a temporary measure to
satisfy the logic in o2net_check_handshake() and will be rectified in a future version of this patch
--- fs/ocfs2/cluster/tcp.c | 44 ++++++++++++++++++++++++++++++++++++++++ fs/ocfs2/cluster/tcp.h | 2 + fs/ocfs2/cluster/tcp_internal.h | 6 ++++- 3 files changed, 51 insertions(+), 1 deletion(-)
ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +44 -0
@@ -1121,6 +1121,33 @@ static int o2net_check_handshake(structreturn -1 ; } +/* +* Ensure timeouts are consistent with other nodes, otherwise +* we can end up with one node thinking that the other must be down, +* but isn't. This can ultimately cause corruption. +*/ +if (be32_to_cpu (hand->o2net_idle_timeout_ms) != +o2net_idle_timeout (sc->sc_node)) { +mlog (ML_NOTICE ,SC_NODEF_FMT " uses network idle timeout of " +"%u ms, but we use %u ms locally. disconnecting\n" , +SC_NODEF_ARGS (sc), +be32_to_cpu (hand->o2net_idle_timeout_ms), +o2net_idle_timeout (sc->sc_node)); +o2net_ensure_shutdown (nn, sc, -ENOTCONN ); +return -1 ; + } + +if (be32_to_cpu (hand->o2net_keepalive_delay_ms) != +o2net_keepalive_delay (sc->sc_node)) { +mlog (ML_NOTICE ,SC_NODEF_FMT " uses keepalive delay of " +"%u ms, but we use %u ms locally. disconnecting\n" , +SC_NODEF_ARGS (sc), +be32_to_cpu (hand->o2net_keepalive_delay_ms), +o2net_keepalive_delay (sc->sc_node)); +o2net_ensure_shutdown (nn, sc, -ENOTCONN ); +return -1 ; + } + sc->sc_handshake_ok =1 ;spin_lock (&nn->nn_lock);
@@ -1269,6 +1296,21 @@ static int o2net_set_nodelay(struct sockreturn ret; } +static void o2net_initialize_handshake (void ) +{ +static int initialized =0 ; +if (initialized) +return ; + + initialized =1 ; + o2net_hand->o2net_idle_timeout_ms =cpu_to_be32 ( +o2net_idle_timeout (NULL )); + o2net_hand->o2net_keepalive_delay_ms =cpu_to_be32 ( +o2net_keepalive_delay (NULL )); + o2net_hand->o2net_reconnect_delay_ms =cpu_to_be32 ( +o2net_reconnect_delay (NULL )); +} +/* ------------------------------------------------------------ */ /* called when a connect completes and after a sock is accepted. the
@@ -1281,6 +1323,7 @@ static void o2net_sc_connect_completed(v (unsigned long long )O2NET_PROTOCOL_VERSION , (unsigned long long)be64_to_cpu(o2net_hand->connector_id)); +o2net_initialize_handshake ();o2net_sendpage (sc, o2net_hand,sizeof (*o2net_hand));sc_put (sc); }
@@ -1668,6 +1711,7 @@ static int o2net_accept_one(struct sockeo2net_register_callbacks (sc->sc_sock->sk, sc);o2net_sc_queue_work (sc, &sc->sc_rx_work); +o2net_initialize_handshake ();o2net_sendpage (sc, o2net_hand,sizeof (*o2net_hand)); out:
ocfs2/cluster/tcp.h renamed from ocfs2/cluster/tcp.h.orig +2 -0
@@ -47,6 +47,8 @@ struct o2net_msg__be32 status;__be32 key;__be32 msg_num; +__be64 dummy; +__be32 dummy2;__u8 buf[0 ]; };
ocfs2/cluster/tcp_internal.h renamed from ocfs2/cluster/tcp_internal.h.orig +5 -1
@@ -48,10 +48,14 @@* - full 64 bit i_size in the metadata lock lvbs * - introduction of "rw" lock and pushing meta/data locking down */ -#define O2NET_PROTOCOL_VERSION 4ULL +#define O2NET_PROTOCOL_VERSION 5ULLstruct o2net_handshake {__be64 protocol_version;__be64 connector_id; +__be32 o2hb_heartbeat_timeout_ms; +__be32 o2net_idle_timeout_ms; +__be32 o2net_keepalive_delay_ms; +__be32 o2net_reconnect_delay_ms; };struct o2net_node {
--
PATCH 2/3 OCFS2 Configurable timeouts
In reply to [email protected]--- fs/ocfs2/cluster/nodemanager.c | 164 ++++++++++++++++++++++++++++++++++++++++ fs/ocfs2/cluster/nodemanager.h | 3 fs/ocfs2/cluster/tcp.c | 60 +++++++++++--- fs/ocfs2/cluster/tcp.h | 7 + fs/ocfs2/cluster/tcp_internal.h | 6 - 5 files changed, 222 insertions(+), 18 deletions(-)
ocfs2/cluster/nodemanager.c renamed from ocfs2/cluster/nodemanager.c.orig +164 -0
@@ -532,6 +532,164 @@ static struct o2nm_node_group *to_o2nm_n }#endif +struct o2nm_cluster_attribute { +struct configfs_attribute attr; +ssize_t (*show)(struct o2nm_cluster *,char *); +ssize_t (*store)(struct o2nm_cluster *,const char *,size_t ); +}; + +static ssize_t o2nm_cluster_attr_write (const char *page,ssize_t count, +unsigned int *val) +{ +unsigned long tmp; +char *p = (char *)page; + + tmp =simple_strtoul (p, &p,0 ); +if (!p || (*p && (*p !='\n' ))) +return -EINVAL ; + +if (tmp ==0 ) +return -EINVAL ; +if (tmp >= (u32)-1 ) +return -ERANGE ; + + *val = tmp; + +return count; +} + +static ssize_t o2nm_cluster_attr_idle_timeout_ms_read (struct o2nm_cluster *cluster, +char *page) +{ +return sprintf (page,"%u\n" , cluster->cl_idle_timeout_ms); +} + +static ssize_t o2nm_cluster_attr_idle_timeout_ms_write (struct o2nm_cluster *cluster, +const char *page, +size_t count) +{ +ssize_t ret; +unsigned int val; + + ret =o2nm_cluster_attr_write (page, count, &val); + +if (ret >0 ) { +if (val <= cluster->cl_keepalive_delay_ms) { +mlog (ML_NOTICE ,"o2net: idle timeout must be larger " +"than keepalive delay\n" ); +return -EINVAL ; + } + cluster->cl_idle_timeout_ms = val; + } + +return ret; +} + +static ssize_t o2nm_cluster_attr_keepalive_delay_ms_read (struct o2nm_cluster *cluster, +char *page) +{ +return sprintf (page,"%u\n" , cluster->cl_keepalive_delay_ms); +} + +static ssize_t o2nm_cluster_attr_keepalive_delay_ms_write (struct o2nm_cluster *cluster, +const char *page, +size_t count) +{ +ssize_t ret; +unsigned int val; + + ret =o2nm_cluster_attr_write (page, count, &val); + +if (ret >0 ) { +if (val >= cluster->cl_idle_timeout_ms) { +mlog (ML_NOTICE ,"o2net: keepalive delay must be " +"smaller than idle timeout\n" ); +return -EINVAL ; + } + cluster->cl_keepalive_delay_ms = val; + } + +return ret; +} + +static ssize_t o2nm_cluster_attr_reconnect_delay_ms_read (struct o2nm_cluster *cluster, +char *page) +{ +return sprintf (page,"%u\n" , cluster->cl_reconnect_delay_ms); +} + +static ssize_t o2nm_cluster_attr_reconnect_delay_ms_write (struct o2nm_cluster *cluster, +const char *page, +size_t count) +{ +return o2nm_cluster_attr_write (page, count, + &cluster->cl_reconnect_delay_ms); +} +static struct o2nm_cluster_attribute o2nm_cluster_attr_idle_timeout_ms = { + .attr = { .ca_owner =THIS_MODULE , + .ca_name ="idle_timeout_ms" , + .ca_mode =S_IRUGO |S_IWUSR }, + .show = o2nm_cluster_attr_idle_timeout_ms_read, + .store = o2nm_cluster_attr_idle_timeout_ms_write, +}; + +static struct o2nm_cluster_attribute o2nm_cluster_attr_keepalive_delay_ms = { + .attr = { .ca_owner =THIS_MODULE , + .ca_name ="keepalive_delay_ms" , + .ca_mode =S_IRUGO |S_IWUSR }, + .show = o2nm_cluster_attr_keepalive_delay_ms_read, + .store = o2nm_cluster_attr_keepalive_delay_ms_write, +}; + +static struct o2nm_cluster_attribute o2nm_cluster_attr_reconnect_delay_ms = { + .attr = { .ca_owner =THIS_MODULE , + .ca_name ="reconnect_delay_ms" , + .ca_mode =S_IRUGO |S_IWUSR }, + .show = o2nm_cluster_attr_reconnect_delay_ms_read, + .store = o2nm_cluster_attr_reconnect_delay_ms_write, +}; + +static struct configfs_attribute *o2nm_cluster_attrs[] = { + &o2nm_cluster_attr_idle_timeout_ms.attr, + &o2nm_cluster_attr_keepalive_delay_ms.attr, + &o2nm_cluster_attr_reconnect_delay_ms.attr, +NULL , +}; +static ssize_t o2nm_cluster_show (struct config_item *item, +struct configfs_attribute *attr, +char *page) +{ +struct o2nm_cluster *cluster =to_o2nm_cluster (item); +struct o2nm_cluster_attribute *o2nm_cluster_attr = + container_of(attr, struct o2nm_cluster_attribute, attr); +ssize_t ret =0 ; + +if (o2nm_cluster_attr->show) + ret = o2nm_cluster_attr->show (cluster, page); +return ret; +} + +static ssize_t o2nm_cluster_store (struct config_item *item, +struct configfs_attribute *attr, +const char *page,size_t count) +{ +struct o2nm_cluster *cluster =to_o2nm_cluster (item); +struct o2nm_cluster_attribute *o2nm_cluster_attr = + container_of(attr, struct o2nm_cluster_attribute, attr); +ssize_t ret; + +if (o2nm_cluster_attr->store ==NULL ) { + ret = -EINVAL ; + goto out; + } + + ret = o2nm_cluster_attr->store (cluster, page, count); +if (ret < count) + goto out; +out: +return ret; +} +static struct config_item *o2nm_node_group_make_item (struct config_group *group,const char *name) {
@@ -613,10 +771,13 @@ static void o2nm_cluster_release(structstatic struct configfs_item_operations o2nm_cluster_item_ops = { .release = o2nm_cluster_release, + .show_attribute = o2nm_cluster_show, + .store_attribute = o2nm_cluster_store, };static struct config_item_type o2nm_cluster_type = { .ct_item_ops = &o2nm_cluster_item_ops, + .ct_attrs = o2nm_cluster_attrs, .ct_owner =THIS_MODULE , };
@@ -667,6 +828,9 @@ static struct config_group *o2nm_cluster cluster->cl_group.default_groups[2 ] =NULL ;rwlock_init (&cluster->cl_nodes_lock); cluster->cl_node_ip_tree =RB_ROOT ; + cluster->cl_reconnect_delay_ms =O2NET_RECONNECT_DELAY_MS_DEFAULT ; + cluster->cl_idle_timeout_ms =O2NET_IDLE_TIMEOUT_MS_DEFAULT ; + cluster->cl_keepalive_delay_ms =O2NET_KEEPALIVE_DELAY_MS_DEFAULT ; ret = &cluster->cl_group; o2nm_single_cluster = cluster;
ocfs2/cluster/nodemanager.h renamed from ocfs2/cluster/nodemanager.h.orig +3 -0
@@ -60,6 +60,9 @@ struct o2nm_cluster {rwlock_t cl_nodes_lock;struct o2nm_node *cl_nodes[O2NM_MAX_NODES ];struct rb_root cl_node_ip_tree; +unsigned int cl_idle_timeout_ms; +unsigned int cl_keepalive_delay_ms; +unsigned int cl_reconnect_delay_ms;/* this bitmap is part of a hack for disk bitmap.. will go eventually. - zab */ unsigned long cl_nodes_bitmap[BITS_TO_LONGS (O2NM_MAX_NODES )];
ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +48 -12
121 lines not on this page · raw message
ocfs2/cluster/tcp.h renamed from ocfs2/cluster/tcp.h.orig +7 -0
@@ -54,6 +54,13 @@ typedef int (o2net_msg_handler_func)(str#define O2NET_MAX_PAYLOAD_BYTES (4096 - sizeof(struct o2net_msg)) +/* same as hb delay, we're waiting for another node to recognize our hb */ +#define O2NET_RECONNECT_DELAY_MS_DEFAULT 2000 + +#define O2NET_KEEPALIVE_DELAY_MS_DEFAULT 5000 +#define O2NET_IDLE_TIMEOUT_MS_DEFAULT 10000 + +/* TODO: figure this out.... */ static inline int o2net_link_down (int err,struct socket *sock) {
ocfs2/cluster/tcp_internal.h renamed from ocfs2/cluster/tcp_internal.h.orig +0 -6
@@ -27,17 +27,11 @@#define O2NET_MSG_KEEP_REQ_MAGIC ((u16)0xfa57)#define O2NET_MSG_KEEP_RESP_MAGIC ((u16)0xfa58) -/* same as hb delay, we're waiting for another node to recognize our hb */ -#define O2NET_RECONNECT_DELAY_MS O2HB_REGION_TIMEOUT_MS -/* we're delaying our quorum decision so that heartbeat will have timed * out truly dead nodes by the time we come around to making decisions * on their number */ #define O2NET_QUORUM_DELAY_MS ((o2hb_dead_threshold + 2) * O2HB_REGION_TIMEOUT_MS) -#define O2NET_KEEPALIVE_DELAY_SECS 5 -#define O2NET_IDLE_TIMEOUT_SECS 10 -/* * This version number represents quite a lot, unfortunately. It not * only represents the raw network message protocol on the wire but also
--
Apologies, quilt seems to have misplaced the comments at the top of each patch:
Patch 1:
From: Andrew Beekhof <[email protected]>
Subject: OCFS2 - Expose struct o2nm_cluster
Subsequent patches (namely userspace heartbeat and configurable timeouts)
require access to the o2nm_cluster struct. This patch does the necessary shuffling.
Patch 2:
From: Jeff Mahoney <[email protected]>
Subject: OCFS2 Configurable timeouts
Allow configuration of OCFS2 timeouts from userspace via configfs
Patch 3:
From: Andrew Beekhof <[email protected]>
Subject: OCFS2 Configurable timeouts - Protocol changes
Modify the OCFS2 handshake to ensure essential timeouts are configured
identically on all nodes.
Included is the field for userspace-heartbeat timeout to avoid the need for
further protocol changes.
The addition of two dummy fields is a temporary measure to
satisfy the logic in o2net_check_handshake() and will be rectified in a future version of this patch
Quoted 5 lines · On Nov 18, 2006, at 8:36 AM, [email protected] wrote:
--
Jeff has asked me to finish off his configurable timeouts patch.... here's the results :-)
Quoted 7 lines
One thing I'm not so happy about are the changes (I made) to
o2net_msgstruct
in order to satisfy the logic ino2net_check_handshake()Presumably the logic in
o2net_check_handshake()should be changed?Andrew
--
Andrew Beekhof
"Would the last person to leave please turn out the enlightenment?" - TISM
PATCH 2/3 OCFS2 Configurable timeouts
In reply to [email protected]Hi Andrew,
Thanks for sending these out. Your first patch looks fine. My
comments for this one are inlined below.
Quoted 43 lines · On Sat, Nov 18, 2006 at 08:36:02AM +0100, [email protected] wrote:
ocfs2/cluster/nodemanager.c renamed from ocfs2/cluster/nodemanager.c.orig +43
-0+static ssize_t o2nm_cluster_attr_idle_timeout_ms_read (struct o2nm_cluster *cluster, +char *page) +{ +return sprintf (page,"%u\n" , cluster->cl_idle_timeout_ms); +} + +static ssize_t o2nm_cluster_attr_idle_timeout_ms_write (struct o2nm_cluster *cluster, +const char *page, +size_t count) +{ +ssize_t ret; +unsigned int val; + + ret =o2nm_cluster_attr_write (page, count, &val); + +if (ret >0 ) { +if (val <= cluster->cl_keepalive_delay_ms) { +mlog (ML_NOTICE ,"o2net: idle timeout must be larger " +"than keepalive delay\n" ); +return -EINVAL ; + } + cluster->cl_idle_timeout_ms = val; + } + +return ret; +} + +static ssize_t o2nm_cluster_attr_keepalive_delay_ms_read (struct o2nm_cluster *cluster, +char *page) +{ +return sprintf (page,"%u\n" , cluster->cl_keepalive_delay_ms); +} + +static ssize_t o2nm_cluster_attr_keepalive_delay_ms_write (struct o2nm_cluster *cluster, +const char *page, +size_t count) +{ +ssize_t ret; +unsigned int val; + + ret =o2nm_cluster_attr_write (page, count, &val); + +if (ret >0 ) {
ocfs2/cluster/nodemanager.c static struct o2nm_node_group *to_o2nm_n · line 613
Quoted 10 lines
ocfs2/cluster/nodemanager.c renamed from ocfs2/cluster/nodemanager.c.orig +10
-0+if (val >= cluster->cl_idle_timeout_ms) { +mlog (ML_NOTICE ,"o2net: keepalive delay must be " +"smaller than idle timeout\n" ); +return -EINVAL ; + } + cluster->cl_keepalive_delay_ms = val; + } + +return ret; +}
Is it safe for us to allow the keepalive delay and the idle timeout values to change while the cluster is up? The next patch should be checking that the values are the same on all nodes, but if we can change it mid stream, what's the point?
Quoted 10 lines
Index:
fs/ocfs2/cluster/tcp.c
===================================================================ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +2
-0@@ -147,6 +147,28 @@ static void o2net_listen_data_ready(strustatic void o2net_sc_send_keep_req (void *arg);static void o2net_idle_timer (unsigned long data);static void o2net_sc_postpone_idle (struct o2net_sock_container *sc); +static void o2net_sc_reset_idle_timer (struct o2net_sock_container *sc); +
ocfs2/cluster/tcp.c static void o2net_listen_data_ready(stru · line 161
Quoted 10 lines
ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +10
-0+/* +* FIXME: These should use to_o2nm_cluster_from_node(), but we end up +* losing our parent link to the cluster during shutdown. This can be +* solved by adding a pre-removal callback to configfs, or passing +* around the cluster with the node. -jeffm +*/ +static inline int o2net_reconnect_delay (struct o2nm_node *node) +{ +return o2nm_single_cluster->cl_reconnect_delay_ms; +}
This patch exists now - it's in ocfs2.git:
You can probably just add a patch on top of all these to fix that to use the new callback.
--Mark
--
Mark Fasheh
Senior Software Developer, Oracle
[email protected]
PATCH 3/3 OCFS2 Configurable timeouts - Protocol changes
In reply to [email protected]Quoted 3 lines · On Sat, Nov 18, 2006 at 08:36:03AM +0100, [email protected] wrote:
The addition of two dummy fields is a temporary measure to
satisfy the logic ino2net_check_handshake()and will be rectified in a future version of this patch
Maybe I'm being silly, but I just can't seem to figure out precisely which logic you're talking about here. Can you elaborate, please?
Quoted 4 lines
ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +4
-0+static void o2net_initialize_handshake (void ) +{ +static int initialized =0 ; +if (initialized)
ocfs2/cluster/tcp.c static int o2net_set_nodelay(struct sock · line 1312
Quoted 10 lines
ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +10
-0+return ; + + initialized =1 ; + o2net_hand->o2net_idle_timeout_ms =cpu_to_be32 ( +o2net_idle_timeout (NULL )); + o2net_hand->o2net_keepalive_delay_ms =cpu_to_be32 ( +o2net_keepalive_delay (NULL )); + o2net_hand->o2net_reconnect_delay_ms =cpu_to_be32 ( +o2net_reconnect_delay (NULL )); +}
We should populate o2hb_heartbeat_timeout_ms here. Checking against it in o2net_check_handshake() would also be good.
Thanks,
--Mark
--
Mark Fasheh
Senior Software Developer, Oracle
[email protected]
PATCH 3/3 OCFS2 Configurable timeouts - Protocol changes
Quoted 7 lines · Mark Fasheh
Quoted 3 lines · On Sat, Nov 18, 2006 at 08:36:03AM +0100, [email protected] wrote:
The addition of two dummy fields is a temporary measure to
satisfy the logic ino2net_check_handshake()and will be rectified in a future version of this patchMaybe I'm being silly, but I just can't seem to figure out precisely which
logic you're talking about here. Can you elaborate, please?
Sorry, I meant the logic that controlls when o2net_check_handshake() is called by o2net_advance_rx().
Specifically, this comment:
/* this working relies on the handshake being * smaller than the normal message header */
Quoted 7 lines
Quoted 7 lines
ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +7
-0+static void o2net_initialize_handshake (void ) +{ +static int initialized =0 ; +if (initialized) +return ; + + initialized =1 ;
Quoted 8 lines
ocfs2/cluster/tcp.c static int o2net_set_nodelay(struct sock · line 1312
Quoted 7 lines
ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +7
-0+ o2net_hand->o2net_idle_timeout_ms =cpu_to_be32 ( +o2net_idle_timeout (NULL )); + o2net_hand->o2net_keepalive_delay_ms =cpu_to_be32 ( +o2net_keepalive_delay (NULL )); + o2net_hand->o2net_reconnect_delay_ms =cpu_to_be32 ( +o2net_reconnect_delay (NULL )); +}We should populate
o2hb_heartbeat_timeout_mshere.
is a constant ok? I left out all other references to heartbeat timeouts.
Quoted 10 lines
Checking against it in
o2net_check_handshake()would also be good.Thanks,
--Mark--
Mark Fasheh
Senior Software Developer, Oracle
[email protected]
--
Andrew Beekhof
"Would the last person to leave please turn out the enlightenment?" - TISM
PATCH 2/3 OCFS2 Configurable timeouts
In reply to Mark FashehQuoted 61 lines · Mark Fasheh
Hi Andrew,
Thanks for sending these out. Your first patch looks fine. My
comments for this one are inlined below.Quoted 56 lines · On Sat, Nov 18, 2006 at 08:36:02AM +0100, [email protected] wrote:
+static
ssize_to2nm_cluster_attr_idle_timeout_ms_read(structo2nm_cluster*cluster,ocfs2/cluster/nodemanager.c renamed from ocfs2/cluster/nodemanager.c.orig +6
-0+ char *page) +{ +return sprintf (page,"%u\n" , cluster->cl_idle_timeout_ms); +} + +static ssize_t o2nm_cluster_attr_idle_timeout_ms_write(struct
o2nm_cluster*cluster,ocfs2/cluster/nodemanager.c renamed from ocfs2/cluster/nodemanager.c.orig +21
-0+const char *page, + size_t count) +{ + ssize_t ret; +unsigned int val; + + ret =o2nm_cluster_attr_write (page, count, &val); + +if (ret >0 ) { +if (val <= cluster->cl_keepalive_delay_ms) { +mlog (ML_NOTICE ,"o2net: idle timeout must be larger " +"than keepalive delay\n" ); +return -EINVAL ; + } + cluster->cl_idle_timeout_ms = val; + } + +return ret; +} + +static ssize_t o2nm_cluster_attr_keepalive_delay_ms_read(struct
o2nm_cluster*cluster,ocfs2/cluster/nodemanager.c renamed from ocfs2/cluster/nodemanager.c.orig +6
-0+ char *page) +{ +return sprintf (page,"%u\n" , cluster->cl_keepalive_delay_ms); +} + +static ssize_t o2nm_cluster_attr_keepalive_delay_ms_write(struct
o2nm_cluster*cluster,ocfs2/cluster/nodemanager.c renamed from ocfs2/cluster/nodemanager.c.orig +18
-0+const char *page, + size_t count) +{ + ssize_t ret; +unsigned int val; + + ret =o2nm_cluster_attr_write (page, count, &val); + +if (ret >0 ) { +if (val >= cluster->cl_idle_timeout_ms) { +mlog (ML_NOTICE ,"o2net: keepalive delay must be " +"smaller than idle timeout\n" ); +return -EINVAL ; + } + cluster->cl_keepalive_delay_ms = val; + } + +return ret;
Quoted 8 lines
Quoted 1 line
+}
Is it safe for us to allow the keepalive delay and the idle timeout values
to change while the cluster is up? The next patch should be checking that
the values are the same on all nodes, but if we can change it mid stream,
what's the point?
good point.
The simplest option would seem to be to lock the values in once we've sent the first handshake.
Maybe one day we can allow nodes to (re-)negotiate the intervals but locking it in is probably enough for now.
Quoted 27 lines
Quoted 21 lines
Index:
fs/ocfs2/cluster/tcp.c
===================================================================ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +4
-0@@ -147,6 +147,28 @@ static void o2net_listen_data_ready(strustatic void o2net_sc_send_keep_req (void *arg);static void o2net_idle_timer (unsigned long data);static void o2net_sc_postpone_idle (struct o2net_sock_container *sc); +static void o2net_sc_reset_idle_timer (struct o2net_sock_container *sc); + +/* +* FIXME: These should use to_o2nm_cluster_from_node(), but we end up
+ * losing our parent link to the cluster during shutdown. This can be
+ * solved by adding a pre-removal callback to configfs, or passing
+ * around the cluster with the node. -jeffm
+ */ocfs2/cluster/tcp.c static void o2net_listen_data_ready(stru · line 161
Quoted 4 lines
ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +4
-0+static inline int o2net_reconnect_delay (struct o2nm_node *node) +{ +return o2nm_single_cluster->cl_reconnect_delay_ms; +}This patch exists now - it's in ocfs2.git:
Quoted 6 lines
http://kernel.org/git/?p=linux/kernel/git/mfasheh/
ocfs2.git;a=commit;h=e47575895914402e1e72f00b3db539f468b849e3You can probably just add a patch on top of all these to fix that to use
the new callback.
will do.
Quoted 6 lines
--Mark--
Mark Fasheh
Senior Software Developer, Oracle
[email protected]
--
Andrew Beekhof
"Would the last person to leave please turn out the enlightenment?" - TISM
PATCH 3/3 OCFS2 Configurable timeouts - Protocol changes
In reply to Andrew BeekhofQuoted 8 lines · Andrew Beekhof
Quoted 7 lines · On Sat, Nov 18, 2006 at 08:36:03AM +0100, [email protected] wrote:
Quoted 3 lines
The addition of two dummy fields is a temporary measure to
satisfy the logic ino2net_check_handshake()and will be rectified in a future version of this patchMaybe I'm being silly, but I just can't seem to figure out precisely which
logic you're talking about here. Can you elaborate, please?
Quoted 6 lines
Sorry, I meant the logic that controlls when
o2net_check_handshake()is called byo2net_advance_rx().Specifically, this comment:
/* this working relies on the handshake being * smaller than the normal message header */
Ahh, ok. Do you have a specific fix in mind? I've CC'd Zach - he might have an idea. Otherwise, I'm not really too worried about growing the o2net_msg structure by a little bit - we ought to rename the fields to something like 'reserved0' and 'reserved1' though.
Quoted 16 lines
Quoted 7 lines
Quoted 7 lines
ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +7
-0+static void o2net_initialize_handshake (void ) +{ +static int initialized =0 ; +if (initialized) +return ; + + initialized =1 ;Quoted 8 lines
ocfs2/cluster/tcp.c static int o2net_set_nodelay(struct sock · line 1312
Quoted 7 lines
ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +7
-0+ o2net_hand->o2net_idle_timeout_ms =cpu_to_be32 ( +o2net_idle_timeout (NULL )); + o2net_hand->o2net_keepalive_delay_ms =cpu_to_be32 ( +o2net_keepalive_delay (NULL )); + o2net_hand->o2net_reconnect_delay_ms =cpu_to_be32 ( +o2net_reconnect_delay (NULL )); +}We should populate
o2hb_heartbeat_timeout_mshere.
Quoted 2 lines
is a constant ok? I left out all other references to heartbeat timeouts.
Why not just populate it with O2HB_MAX_WRITE_TIMEOUT_MS? o2hb_dead_threshold is already not allowed to be changed once the cluster is up btw.
--Mark
--
Mark Fasheh
Senior Software Developer, Oracle
[email protected]
PATCH 2/3 OCFS2 Configurable timeouts
In reply to Andrew BeekhofQuoted 9 lines · Andrew Beekhof
Quoted 7 lines · Mark Fasheh
Is it safe for us to allow the keepalive delay and the idle timeout values
to change while the cluster is up? The next patch should be checking that
the values are the same on all nodes, but if we can change it mid stream,
what's the point?
Quoted 5 lines
good point.
The simplest option would seem to be to lock the values in once we've sent the first handshake.
Maybe one day we can allow nodes to (re-)negotiate the intervals but locking it in is probably enough for now.
That sounds good to me.
--Mark
--
Mark Fasheh
Senior Software Developer, Oracle
[email protected]
PATCH 3/3 OCFS2 Configurable timeouts - Protocol changes
In reply to Mark FashehQuoted 9 lines · Mark Fasheh
Quoted 8 lines · Andrew Beekhof
Quoted 7 lines · On Sat, Nov 18, 2006 at 08:36:03AM +0100, [email protected] wrote:
Quoted 3 lines
The addition of two dummy fields is a temporary measure to
satisfy the logic ino2net_check_handshake()and will be rectified in a future version of this patchMaybe I'm being silly, but I just can't seem to figure out precisely which
logic you're talking about here. Can you elaborate, please?
Quoted 7 lines
Quoted 6 lines
Sorry, I meant the logic that controlls when
o2net_check_handshake()is called byo2net_advance_rx().Specifically, this comment:
/* this working relies on the handshake being * smaller than the normal message header */Ahh, ok. Do you have a specific fix in mind?
not yet - i was mostly worried about making it work (and understanding why) so that I could get the rest out for review.
Quoted 5 lines
I've CC'd Zach - he
might have an idea. Otherwise, I'm not really too worried about growing theo2net_msgstructure by a little bit - we ought to rename the fields to something like 'reserved0' and 'reserved1' though.
chuckle... "what's in a name"
Quoted 18 lines
Quoted 18 lines
Quoted 17 lines
Quoted 6 lines
ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +4
-0+static void o2net_initialize_handshake (void ) +{ +static int initialized =0 ; +if (initialized)ocfs2/cluster/tcp.c static int o2net_set_nodelay(struct sock · line 1312
Quoted 10 lines
ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +10
-0+return ; + + initialized =1 ; + o2net_hand->o2net_idle_timeout_ms =cpu_to_be32 ( +o2net_idle_timeout (NULL )); + o2net_hand->o2net_keepalive_delay_ms =cpu_to_be32 ( +o2net_keepalive_delay (NULL )); + o2net_hand->o2net_reconnect_delay_ms =cpu_to_be32 ( +o2net_reconnect_delay (NULL )); +}We should populate
o2hb_heartbeat_timeout_mshere.
Quoted 3 lines
Quoted 2 lines
is a constant ok? I left out all other references to heartbeat timeouts.
Why not just populate it with
O2HB_MAX_WRITE_TIMEOUT_MS?
ok
Quoted 2 lines
o2hb_dead_threshold
is already not allowed to be changed once the cluster is up btw.
ok
--
Andrew Beekhof
"Would the last person to leave please turn out the enlightenment?" - TISM
PATCH 3/3 OCFS2 Configurable timeouts - Protocol changes
In reply to Mark FashehQuoted 11 lines
Quoted 7 lines
Sorry, I meant the logic that controlls when
o2net_check_handshake()is called byo2net_advance_rx().Specifically, this comment:
/* this working relies on the handshake being * smaller than the normal message header */Ahh, ok. Do you have a specific fix in mind? I've CC'd Zach - he might have an idea. Otherwise, I'm not really too worried about growing the
o2net_msgstructure by a little bit - we ought to rename the fields to something like 'reserved0' and 'reserved1' though.
The current o2net_advance_rx() is being lazy and is putting the check for handshake negotiation inside the block which receives the message header. The current code will mistake a partially received handshake header for a msg header once it exceeds the size of the message header.
The fix is easy, just hoist the handshake checking up into it's own block before the header is received. It'd look like the header receiving loop, but would call _check_handshake(). It needs to be careful not to keep trying to parse headers if it doesn't get the whole handshake. So something like:
if (unlikely(!handshake_ok)) {
setup data and datalen, recv into them;
if +ve, inc page_off
if (page_off == sizeof(handshake))
ret = _check_handshake()
if (!handshake_ok)
goto out;
}You'd want to think harder about the return codes than I have here.
- z