OCFS Configurable timeouts - Revision 2
cover OCFS Configurable timeouts - Revision 2
Hopefully this is the final version :-)
--
Added a global spinlock around modifications to o2net_connected_peer (as discussed with Mark).
I have a separate patch that uses to_o2nm_cluster_from_node() but since I cant reproduce the problem mentioned in Jeff's comment (it apparently needs the userspace heartbeating modifications), I'd prefer to leave it out.
Andrew
PATCH 1/1 ocfs2-timeout-protocol.patch
From: Andrew Beekhof <[email protected]>
Subject: [patch 1/1] OCFS2 Configurable timeouts - Protocol changes
Modify the OCFS2 handshake to ensure essential timeouts are configured
identically on all nodes.
Only allow changes when there are no connected peers
Improves the logic in o2net_advance_rx() which broke now that
sizeof(struct o2net_handshake) is greater than sizeof(struct o2net_msg)
Included is the field for userspace-heartbeat timeout to avoid the need for
further protocol changes.
Uses a global spinlock to ensure the decisions to update configfs entries
are made on the correct value. The region covered by the spinlock when incrimenting the counter is much larger as this is the more critical case.
--- fs/ocfs2/cluster/nodemanager.c | 19 +++++++ fs/ocfs2/cluster/tcp.c | 96 +++++++++++++++++++++++++++++++++++----- fs/ocfs2/cluster/tcp.h | 1 fs/ocfs2/cluster/nodemanager.c | 55 ++++++++++++++------ fs/ocfs2/cluster/tcp.c | 105 +++++++++++++++++++++++++++++++++++----- fs/ocfs2/cluster/tcp.h | 2 fs/ocfs2/cluster/tcp_internal.h | 6 +- 4 files changed, 139 insertions(+), 29 deletions(-)
ocfs2/cluster/nodemanager.c renamed from ocfs2/cluster/nodemanager.c.orig +38 -17
@@ -558,15 +558,14 @@ static ssize_t o2nm_cluster_attr_write(creturn count; } -static ssize_t o2nm_cluster_attr_idle_timeout_ms_read (struct o2nm_cluster *cluster, -char *page) +static ssize_t o2nm_cluster_attr_idle_timeout_ms_read ( +struct o2nm_cluster *cluster,char *page) {return sprintf (page,"%u\n" , cluster->cl_idle_timeout_ms); } -static ssize_t o2nm_cluster_attr_idle_timeout_ms_write (struct o2nm_cluster *cluster, -const char *page, -size_t count) +static ssize_t o2nm_cluster_attr_idle_timeout_ms_write ( +struct o2nm_cluster *cluster,const char *page,size_t count) {ssize_t ret;unsigned int val;
@@ -574,10 +573,22 @@ static ssize_t o2nm_cluster_attr_idle_ti ret =o2nm_cluster_attr_write (page, count, &val);if (ret >0 ) { +if (cluster->cl_idle_timeout_ms != val) { +spin_lock (&connected_lock); +if (o2net_num_connected_peers ()) { +mlog (ML_NOTICE , +"o2net: cannot change idle timeout after " +"the first peer has agreed to it." +" %d connected peers\n" , +o2net_num_connected_peers ()); + ret = -EINVAL ; + } +spin_unlock (&connected_lock); + }if (val <= cluster->cl_keepalive_delay_ms) {mlog (ML_NOTICE ,"o2net: idle timeout must be larger " "than keepalive delay\n" ); -return -EINVAL ; + ret = -EINVAL ; } cluster->cl_idle_timeout_ms = val; }
@@ -585,15 +596,14 @@ static ssize_t o2nm_cluster_attr_idle_tireturn ret; } -static ssize_t o2nm_cluster_attr_keepalive_delay_ms_read (struct o2nm_cluster *cluster, -char *page) +static ssize_t o2nm_cluster_attr_keepalive_delay_ms_read ( +struct o2nm_cluster *cluster,char *page) {return sprintf (page,"%u\n" , cluster->cl_keepalive_delay_ms); } -static ssize_t o2nm_cluster_attr_keepalive_delay_ms_write (struct o2nm_cluster *cluster, -const char *page, -size_t count) +static ssize_t o2nm_cluster_attr_keepalive_delay_ms_write ( +struct o2nm_cluster *cluster,const char *page,size_t count) {ssize_t ret;unsigned int val;
@@ -601,10 +611,22 @@ static ssize_t o2nm_cluster_attr_keepali ret =o2nm_cluster_attr_write (page, count, &val);if (ret >0 ) { +if (cluster->cl_keepalive_delay_ms != val) { +spin_lock (&connected_lock); +if (o2net_num_connected_peers ()) { +mlog (ML_NOTICE , +"o2net: cannot change keepalive delay after" +" the first peer has agreed to it." +" %d connected peers\n" , +o2net_num_connected_peers ()); + ret = -EINVAL ; + } +spin_unlock (&connected_lock); + }if (val >= cluster->cl_idle_timeout_ms) {mlog (ML_NOTICE ,"o2net: keepalive delay must be " "smaller than idle timeout\n" ); -return -EINVAL ; + ret = -EINVAL ; } cluster->cl_keepalive_delay_ms = val; }
@@ -612,15 +634,14 @@ static ssize_t o2nm_cluster_attr_keepalireturn ret; } -static ssize_t o2nm_cluster_attr_reconnect_delay_ms_read (struct o2nm_cluster *cluster, -char *page) +static ssize_t o2nm_cluster_attr_reconnect_delay_ms_read ( +struct o2nm_cluster *cluster,char *page) {return sprintf (page,"%u\n" , cluster->cl_reconnect_delay_ms); } -static ssize_t o2nm_cluster_attr_reconnect_delay_ms_write (struct o2nm_cluster *cluster, -const char *page, -size_t count) +static ssize_t o2nm_cluster_attr_reconnect_delay_ms_write ( +struct o2nm_cluster *cluster,const char *page,size_t count) {return o2nm_cluster_attr_write (page, count, &cluster->cl_reconnect_delay_ms);
ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +94 -11
189 lines not on this page · raw message
ocfs2/cluster/tcp.h renamed from ocfs2/cluster/tcp.h.orig +2 -0
@@ -103,11 +103,13 @@ int o2net_register_handler(u32 msg_type,void o2net_unregister_handler_list (struct list_head *list);struct o2nm_node ; +extern spinlock_t connected_lock;int o2net_register_hb_callbacks (void );void o2net_unregister_hb_callbacks (void );int o2net_start_listening (struct o2nm_node *node);void o2net_stop_listening (struct o2nm_node *node);void o2net_disconnect_node (struct o2nm_node *node); +int o2net_num_connected_peers (void );int o2net_init (void );void o2net_exit (void );
ocfs2/cluster/tcp_internal.h renamed from ocfs2/cluster/tcp_internal.h.orig +5 -1
@@ -48,10 +48,14 @@* - full 64 bit i_size in the metadata lock lvbs * - introduction of "rw" lock and pushing meta/data locking down */ -#define O2NET_PROTOCOL_VERSION 4ULL +#define O2NET_PROTOCOL_VERSION 5ULLstruct o2net_handshake {__be64 protocol_version;__be64 connector_id; +__be32 o2hb_heartbeat_timeout_ms; +__be32 o2net_idle_timeout_ms; +__be32 o2net_keepalive_delay_ms; +__be32 o2net_reconnect_delay_ms; };struct o2net_node {
--
PATCH 1/1 ocfs2-timeout-protocol.patch
Quoted 2 lines
Only allow changes when there are no connected peers
I think we can do this differently.
ocfs2/cluster/nodemanager.c static ssize_t o2nm_cluster_attr_idle_ti · line 587
Quoted 13 lines
ocfs2/cluster/nodemanager.c renamed from ocfs2/cluster/nodemanager.c.orig +12
-0if (ret >0 ) { +if (cluster->cl_idle_timeout_ms != val) { +spin_lock (&connected_lock); +if (o2net_num_connected_peers ()) { +mlog (ML_NOTICE , +"o2net: cannot change idle timeout after " +"the first peer has agreed to it." +" %d connected peers\n" , +o2net_num_connected_peers ()); + ret = -EINVAL ; + } +spin_unlock (&connected_lock); + }
Lose the locking here so this just becomes (paraphrasing) :
if (cluster != val && connected()) {
...
}ocfs2/cluster/tcp.c static void o2net_still_up(void *arg) · line 1551
Quoted 7 lines
ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +7
-0+static int o2net_connected_peers =0 ; +spinlock_t connected_lock; + +int o2net_num_connected_peers (void ) +{ +return o2net_connected_peers; +}
Make this an "atomic_t o2net_connected_peers = ATOMIC_INIT(0);" and then "return atomic_read();". We probably don't really need a heavy-weight atomic_t, but it's trivial and this isn't a fast path.
ocfs2/cluster/tcp.c static void o2net_still_up(void *arg) · line 1561
Quoted 9 lines
void
o2net_disconnect_node(struct o2nm_node *node){ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +4
-0struct o2net_node *nn =o2net_nn_from_num (node->nd_num);/* don't reconnect until it's heartbeating again */ +spin_lock (&connected_lock); + o2net_connected_peers--; +spin_unlock (&connected_lock); +
Then don't do this work here and in other places, do it once in o2net_set_nn_state(). something like:
if (old_sc != sc) {
if (old_sc)
atomic_dec(&o2net_connected_peers);
else
atomic_inc(&o2net_connected_peers);
}That's less confusing and catches the place where sockets are in use by a node.
- z
PATCH 1/1 ocfs2-timeout-protocol.patch
In reply to [email protected]Hi Andrew,
Things are looking much better, but there's still a few issues that
I found while reviewing the patch. I got Zach to look at it too (he's the original author of the ocfs2 network code) which has generated some good comments.
Quoted 4 lines · On Wed, Nov 29, 2006 at 09:51:31AM +0100, [email protected] wrote:
From: Andrew Beekhof <[email protected]>
Subject: [patch 1/1] OCFS2 Configurable timeouts - Protocol changes
Quoted 10 lines
Modify the OCFS2 handshake to ensure essential timeouts are configured
identically on all nodes.
Only allow changes when there are no connected peers
Improves the logic ino2net_advance_rx()which broke now that
sizeof(struct o2net_handshake)is greater thansizeof(struct o2net_msg)
Included is the field for userspace-heartbeat timeout to avoid the need for
further protocol changes.
Uses a global spinlock to ensure the decisions to update configfs entries
are made on the correct value. The region covered by the spinlock when incrimenting the counter is much larger as this is the more critical case.
Nitpick: Can you format that commit log to be a bit more in line with standard kernel commits (the indenting is weird)
Quoted 5 lines
Index:
fs/ocfs2/cluster/nodemanager.c
===================================================================
ocfs2/cluster/nodemanager.c static ssize_t o2nm_cluster_attr_write(c · line 565
Quoted 10 lines
ocfs2/cluster/nodemanager.c renamed from ocfs2/cluster/nodemanager.c.orig +2
-2@@ -558,15 +558,14 @@ static ssize_t o2nm_cluster_attr_write(creturn count; } -static ssize_t o2nm_cluster_attr_idle_timeout_ms_read (struct o2nm_cluster *cluster, -char *page) +static ssize_t o2nm_cluster_attr_idle_timeout_ms_read ( +struct o2nm_cluster *cluster,char *page) {return sprintf (page,"%u\n" , cluster->cl_idle_timeout_ms); }
Can you not re-write the function prototypes unless they're actually changing please? It clutters up the patch and makes it harder to find the actual code to check (see below).
Quoted 13 lines
ocfs2/cluster/nodemanager.c renamed from ocfs2/cluster/nodemanager.c.orig +9
-0@@ -574,10 +573,22 @@ static ssize_t o2nm_cluster_attr_idle_ti ret =o2nm_cluster_attr_write (page, count, &val);if (ret >0 ) { +if (cluster->cl_idle_timeout_ms != val) { +spin_lock (&connected_lock); +if (o2net_num_connected_peers ()) { +mlog (ML_NOTICE , +"o2net: cannot change idle timeout after " +"the first peer has agreed to it." +" %d connected peers\n" , +o2net_num_connected_peers ()); + ret = -EINVAL ;
ocfs2/cluster/nodemanager.c static ssize_t o2nm_cluster_attr_idle_ti · line 593
Quoted 10 lines
ocfs2/cluster/nodemanager.c renamed from ocfs2/cluster/nodemanager.c.orig +4
-1+ } +spin_unlock (&connected_lock); + }if (val <= cluster->cl_keepalive_delay_ms) {mlog (ML_NOTICE ,"o2net: idle timeout must be larger " "than keepalive delay\n" ); -return -EINVAL ; + ret = -EINVAL ; } cluster->cl_idle_timeout_ms = val;
I don't know how I missed this before, but you're erroring with a negative return value, yet continuing with the work of setting cluster->cl_idle_timeout_ms anyway. I think we're missing some goto's here and in the similar blocks below.
Quoted 27 lines
ocfs2/cluster/nodemanager.c renamed from ocfs2/cluster/nodemanager.c.orig +23
-0@@ -1121,6 +1121,44 @@ static int o2net_check_handshake(structreturn -1 ; } +/* +* Ensure timeouts are consistent with other nodes, otherwise +* we can end up with one node thinking that the other must be down, +* but isn't. This can ultimately cause corruption. +*/ +if (be32_to_cpu (hand->o2net_idle_timeout_ms) != +o2net_idle_timeout (sc->sc_node)) { +mlog (ML_NOTICE ,SC_NODEF_FMT " uses a network idle timeout of " +"%u ms, but we use %u ms locally. disconnecting\n" , +SC_NODEF_ARGS (sc), +be32_to_cpu (hand->o2net_idle_timeout_ms), +o2net_idle_timeout (sc->sc_node)); +o2net_ensure_shutdown (nn, sc, -ENOTCONN ); +return -1 ; + } + +if (be32_to_cpu (hand->o2net_keepalive_delay_ms) != +o2net_keepalive_delay (sc->sc_node)) { +mlog (ML_NOTICE ,SC_NODEF_FMT " uses a keepalive delay of " +"%u ms, but we use %u ms locally. disconnecting\n" , +SC_NODEF_ARGS (sc), +be32_to_cpu (hand->o2net_keepalive_delay_ms), +o2net_keepalive_delay (sc->sc_node));
ocfs2/cluster/nodemanager.c static int o2net_check_handshake(struct · line 1156
Quoted 10 lines
ocfs2/cluster/nodemanager.c renamed from ocfs2/cluster/nodemanager.c.orig +10
-0+o2net_ensure_shutdown (nn, sc, -ENOTCONN ); +return -1 ; + } + +if (be32_to_cpu (hand->o2hb_heartbeat_timeout_ms) != +O2HB_MAX_WRITE_TIMEOUT_MS ) { + mlog(ML_NOTICE, SC_NODEF_FMT " uses a heartbeat timeout of " + "%u ms, but we use %u ms locally. disconnecting\n", + SC_NODEF_ARGS(sc), + be32_to_cpu(hand->o2net_keepalive_delay_ms),
We check hearbeat timeout here, but print keepalive delay...
Quoted 8 lines
ocfs2/cluster/nodemanager.c renamed from ocfs2/cluster/nodemanager.c.orig +4
-0@@ -1153,6 +1191,26 @@ static int o2net_advance_rx(struct o2netsclog (sc,"receiving\n" );do_gettimeofday (&sc->sc_tv_advance_start); +if (unlikely (sc->sc_handshake_ok ==0 )) { +if (sc->sc_page_off <sizeof (struct o2net_handshake )) { + data =page_address (sc->sc_page) + sc->sc_page_off; + datalen =sizeof (struct o2net_handshake ) - sc->sc_page_off;
ocfs2/cluster/nodemanager.c static int o2net_advance_rx(struct o2net · line 1207
Quoted 10 lines
ocfs2/cluster/nodemanager.c renamed from ocfs2/cluster/nodemanager.c.orig +10
-0+ ret =o2net_recv_tcp_msg (sc->sc_sock, data, datalen); +if (ret >0 ) + sc->sc_page_off += ret; + } + +if (sc->sc_page_off ==sizeof (struct o2net_handshake )) { +o2net_check_handshake (sc); +if (sc->sc_handshake_ok ==0 ) { +BUG_ON (sizeof (struct o2net_handshake ) + ==sizeof (struct o2net_msg ));
Is this necessary? Didn't we fix the logic such that the relative sizes don't matter any more? If it _is_ necessary, then it should be a BUILD_BUG_ON() in a more visible place, with a nice fat comment explaining why...
ocfs2/cluster/nodemanager.c static int o2net_advance_rx(struct o2net · line 1210
Quoted 3 lines
ocfs2/cluster/nodemanager.c renamed from ocfs2/cluster/nodemanager.c.orig +3
-0+ ret = -EPROTO ; + } + goto out;
Do you mean to move that goto within the
if (sc->sc_handshake_ok == 0) {
block? I _think_ it's ok for us to continue otherwise...
ocfs2/cluster/nodemanager.c static int o2net_advance_rx(struct o2net · line 1232
Quoted 9 lines
ocfs2/cluster/nodemanager.c renamed from ocfs2/cluster/nodemanager.c.orig +1
-2@@ -1178,8 +1227,7 @@ static int o2net_advance_rx(struct o2net O2NET_MAX_PAYLOAD_BYTES) ret = -EOVERFLOW ; } - } -if (ret <=0 ) + }else goto out; }
Why are you doing that? We'll continue now if we want to return -EOVERFLOW where we would error out before.
Thanks,
--Mark
--
Mark Fasheh
Senior Software Developer, Oracle
[email protected]
PATCH 1/1 ocfs2-timeout-protocol.patch
Quoted 29 lines · Mark Fasheh
Hi Andrew,
Things are looking much better, but there's still a few issues that
I found while reviewing the patch. I got Zach to look at it too (he's the
original author of the ocfs2 network code) which has generated some good
comments.Quoted 4 lines · On Wed, Nov 29, 2006 at 09:51:31AM +0100, [email protected] wrote:
From: Andrew Beekhof <[email protected]>
Subject: [patch 1/1] OCFS2 Configurable timeouts - Protocol changesQuoted 16 lines
Modify the OCFS2 handshake to ensure essential timeouts are configured
identically on all nodes.
Only allow changes when there are no connected peers Improves the logic ino2net_advance_rx()which broke now that
sizeof(struct o2net_handshake)is greater than sizeof(structo2net_msg)
Included is the field for userspace-heartbeat timeout to avoid the need for
further protocol changes.
Uses a global spinlock to ensure the decisions to update configfs entries
are made on the correct value. The region covered by the
spinlock when
incrimenting the counter is much larger as this is the more
critical case.
Quoted 2 lines
Nitpick: Can you format that commit log to be a bit more in line with standard kernel commits (the indenting is weird)
sure
Quoted 12 lines
Quoted 12 lines
Index:
fs/ocfs2/cluster/nodemanager.c
===================================================================
---fs/ocfs2/cluster/nodemanager.c.orig2006-11-20
16:25:58.000000000 +0100
+++fs/ocfs2/cluster/nodemanager.c2006-11-27 09:57:56.000000000
+0100ocfs2/cluster/nodemanager.c renamed from ocfs2/cluster/nodemanager.c.orig +0
-0@@ -558,15 +558,14 @@ static ssize_t o2nm_cluster_attr_write(creturn count; }
Quoted 12 lines
ocfs2/cluster/nodemanager.c static ssize_t o2nm_cluster_attr_write(c · line 565
Quoted 8 lines
ocfs2/cluster/nodemanager.c renamed from ocfs2/cluster/nodemanager.c.orig +2
-2-static ssize_t o2nm_cluster_attr_idle_timeout_ms_read (struct o2nm_cluster *cluster, -char *page) +static ssize_t o2nm_cluster_attr_idle_timeout_ms_read( + struct o2nm_cluster *cluster, char *page) { return sprintf(page, "%u\n", cluster->cl_idle_timeout_ms); }Can you not re-write the function prototypes unless they're actually changing please? It clutters up the patch and makes it harder to find the
actual code to check (see below).
ah, bad habit i picked up working on smaller projects. is it ok in a separate patch? or have I got my wrap points set too small by default?
Quoted 28 lines
Quoted 15 lines
ocfs2/cluster/nodemanager.c renamed from ocfs2/cluster/nodemanager.c.orig +9
-0@@ -574,10 +573,22 @@ static ssize_t o2nm_cluster_attr_idle_ti ret =o2nm_cluster_attr_write (page, count, &val);if (ret >0 ) { +if (cluster->cl_idle_timeout_ms != val) { +spin_lock (&connected_lock); +if (o2net_num_connected_peers ()) { +mlog (ML_NOTICE , +"o2net: cannot change idle timeout after " +"the first peer has agreed to it." +" %d connected peers\n" , +o2net_num_connected_peers ()); + ret = -EINVAL ;ocfs2/cluster/nodemanager.c static ssize_t o2nm_cluster_attr_idle_ti · line 593
Quoted 10 lines
ocfs2/cluster/nodemanager.c renamed from ocfs2/cluster/nodemanager.c.orig +4
-1+ } +spin_unlock (&connected_lock); + }if (val <= cluster->cl_keepalive_delay_ms) {mlog (ML_NOTICE ,"o2net: idle timeout must be larger " "than keepalive delay\n" ); -return -EINVAL ; + ret = -EINVAL ; } cluster->cl_idle_timeout_ms = val;I don't know how I missed this before, but you're erroring with a negative return
value, yet continuing with the work of setting cluster-
Quoted 4 lines
Quoted 1 line
cl_idle_timeout_msanyway. I think we're missing some goto's here and in the similar blocks
below.
my bad - fixed
Quoted 32 lines
Quoted 32 lines
ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +26
-0@@ -1121,6 +1121,44 @@ static int o2net_check_handshake(structreturn -1 ; } +/* +* Ensure timeouts are consistent with other nodes, otherwise +* we can end up with one node thinking that the other must be down, +* but isn't. This can ultimately cause corruption. +*/ +if (be32_to_cpu (hand->o2net_idle_timeout_ms) != +o2net_idle_timeout (sc->sc_node)) { +mlog (ML_NOTICE ,SC_NODEF_FMT " uses a network idle timeout of " +"%u ms, but we use %u ms locally. disconnecting\n" , +SC_NODEF_ARGS (sc), +be32_to_cpu (hand->o2net_idle_timeout_ms), +o2net_idle_timeout (sc->sc_node)); +o2net_ensure_shutdown (nn, sc, -ENOTCONN ); +return -1 ; + } + +if (be32_to_cpu (hand->o2net_keepalive_delay_ms) != +o2net_keepalive_delay (sc->sc_node)) { +mlog (ML_NOTICE ,SC_NODEF_FMT " uses a keepalive delay of " +"%u ms, but we use %u ms locally. disconnecting\n" , +SC_NODEF_ARGS (sc), +be32_to_cpu (hand->o2net_keepalive_delay_ms), +o2net_keepalive_delay (sc->sc_node)); +o2net_ensure_shutdown (nn, sc, -ENOTCONN ); +return -1 ; + }
Quoted 9 lines
ocfs2/cluster/tcp.c static int o2net_check_handshake(struct · line 1156
Quoted 7 lines
ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +7
-0+ +if (be32_to_cpu (hand->o2hb_heartbeat_timeout_ms) != +O2HB_MAX_WRITE_TIMEOUT_MS ) { + mlog(ML_NOTICE, SC_NODEF_FMT " uses a heartbeat timeout of " + "%u ms, but we use %u ms locally. disconnecting\n", + SC_NODEF_ARGS(sc), + be32_to_cpu(hand->o2net_keepalive_delay_ms),We check hearbeat timeout here, but print keepalive delay...
fixed
Quoted 5 lines
Quoted 5 lines
ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +0
-0@@ -1153,6 +1191,26 @@ static int o2net_advance_rx(struct o2netsclog (sc,"receiving\n" );do_gettimeofday (&sc->sc_tv_advance_start);
Quoted 15 lines
Quoted 4 lines
ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +4
-0+if (unlikely (sc->sc_handshake_ok ==0 )) { +if (sc->sc_page_off <sizeof (struct o2net_handshake )) { + data =page_address (sc->sc_page) + sc->sc_page_off; + datalen =sizeof (struct o2net_handshake ) - sc->sc_page_off;ocfs2/cluster/tcp.c static int o2net_advance_rx(struct o2net · line 1207
Quoted 10 lines
ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +10
-0+ ret =o2net_recv_tcp_msg (sc->sc_sock, data, datalen); +if (ret >0 ) + sc->sc_page_off += ret; + } + +if (sc->sc_page_off ==sizeof (struct o2net_handshake )) { +o2net_check_handshake (sc); +if (sc->sc_handshake_ok ==0 ) { +BUG_ON (sizeof (struct o2net_handshake ) + ==sizeof (struct o2net_msg ));Is this necessary?
I wasnt sure at the time - see below - so i wanted to make sure it at least died sanely
apparently i still needed education on how that is done :)
Quoted 3 lines
Didn't we fix the logic such that the relative sizes don't matter any more? If it _is_ necessary, then it should be a
BUILD_BUG_ON()in a more visible place,
ah, I was not familiar with that macro yet
Quoted 11 lines
with a nice fat comment explaining
why...ocfs2/cluster/tcp.c static int o2net_advance_rx(struct o2net · line 1210
Quoted 3 lines
ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +3
-0+ ret = -EPROTO ; + } + goto out;Do you mean to move that goto within the
if (
sc->sc_handshake_ok== 0) {block? I _think_ it's ok for us to continue otherwise...
i did - but if we never want to process an o2net_msg if the handshake has not been completed, then i can structure things a little differently/clearly
Quoted 14 lines
ocfs2/cluster/tcp.c static int o2net_advance_rx(struct o2net · line 1232
Quoted 11 lines
ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +1
-2@@ -1178,8 +1227,7 @@ static int o2net_advance_rx(struct o2net O2NET_MAX_PAYLOAD_BYTES) ret = -EOVERFLOW ; } - } -if (ret <=0 ) + }else goto out; }Why are you doing that? We'll continue now if we want to return - EOVERFLOW
where we would error out before.
damn - i should have noticed that
I'll resubmit once we sort out the o2net_msg / o2net_handshake situation
--
Andrew Beekhof
"Would the last person to leave please turn out the enlightenment?" - TISM
PATCH 1/1 ocfs2-timeout-protocol.patch
Quoted 8 lines · Andrew Beekhof
Quoted 4 lines
Can you not re-write the function prototypes unless they're actually changing please? It clutters up the patch and makes it harder to find the
actual code to check (see below).ah, bad habit i picked up working on smaller projects. is it ok in a separate patch? or have I got my wrap points set too small by default?
In general if you see something that's inconsisent with Documentation/CodingStyle, then yeah fixing it in a seperate patch is fine.
As far as those functions are concerned, if you're not happy with the indentation, then it would be nicest if you got them right in the patch which introduced them.
Quoted 11 lines
Quoted 6 lines
ocfs2/cluster/tcp.c static int o2net_advance_rx(struct o2net · line 1207
Quoted 5 lines
ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +5
-0+if (sc->sc_page_off ==sizeof (struct o2net_handshake )) { +o2net_check_handshake (sc); +if (sc->sc_handshake_ok ==0 ) { +BUG_ON (sizeof (struct o2net_handshake ) + ==sizeof (struct o2net_msg ));Is this necessary?
I wasnt sure at the time - see below - so i wanted to make sure it at least died sanely
apparently i still needed education on how that is done :)
Quoted 5 lines
Quoted 3 lines
Didn't we fix the logic such that the relative sizes don't matter any more? If it _is_ necessary, then it should be a
BUILD_BUG_ON()in a more visible place,ah, I was not familiar with that macro yet
Yeah, I'm mostly going on your description of the patch, and Zach's description of the problem. I'll have to look more closely to see if this is still something we need to trap or not.
Quoted 8 lines
Quoted 8 lines
with a nice fat comment explaining
why...ocfs2/cluster/tcp.c static int o2net_advance_rx(struct o2net · line 1210
Quoted 3 lines
ocfs2/cluster/tcp.c renamed from ocfs2/cluster/tcp.c.orig +3
-0+ ret = -EPROTO ; + } + goto out;Do you mean to move that goto within the
Quoted 7 lines
Quoted 3 lines
if (
sc->sc_handshake_ok== 0) {block? I _think_ it's ok for us to continue otherwise...
i did - but if we never want to process an
o2net_msgif the handshake has not been completed, then i can structure things a little differently/clearly
Hmm, yeah... It looks like what'd happen if we don't get a properly sized handshake is that we'd continue to process the o2net_msg, which I don't think we want to do. If we skipped that, then we don't have to depend on the sizes not matching... Which is fine because I don't think we ought to be processing messages from nodes which haven't properly connected to us yet.
Quoted 2 lines
I'll resubmit once we sort out the
o2net_msg/o2net_handshakesituation
Great, thanks alot Andrew!
--Mark
--
Mark Fasheh
Senior Software Developer, Oracle
[email protected]
cover OCFS Configurable timeouts - Revision 2
In reply to [email protected]Quoted 3 lines · On Wed, Nov 29, 2006 at 09:51:30AM +0100, [email protected] wrote:
Added a global spinlock around modifications to
o2net_connected_peer(as discussed with Mark).I have a separate patch that uses
to_o2nm_cluster_from_node()but since I cant reproduce the problem mentioned in Jeff's comment (it apparently needs the userspace heartbeating modifications), I'd prefer to leave it out.
Which problem do you mean? I'm trying to know if the callback
changes (->disconnect_notify()) make you happy and are good enough to push towards mainline. Have you tested with them, etc?
Joel
Signature · 9 lines
Bram's Law: The easier a piece of software is to write, the worse it's implemented in practice. Joel Becker Principal Software Developer Oracle E-mail: [email protected] Phone: (650) 506-8127
cover OCFS Configurable timeouts - Revision 2
Quoted 4 lines · Joel Becker
Quoted 3 lines · On Wed, Nov 29, 2006 at 09:51:30AM +0100, [email protected] wrote:
Added a global spinlock around modifications to
o2net_connected_peer(as discussed with Mark).
Quoted 8 lines
Quoted 4 lines
I have a separate patch that uses
to_o2nm_cluster_from_node()but since I cant reproduce the problem mentioned in Jeff's comment (it apparently needs the userspace heartbeating modifications), I'd prefer to leave it out.Which problem do you mean? I'm trying to know if the callback
changes (->disconnect_notify()) make you happy and are good enough to push towards mainline. Have you tested with them, etc?
I'm referring to this problem noted by Jeff
/*
* FIXME: These should use to_o2nm_cluster_from_node(), but we end up
* losing our parent link to the cluster during shutdown. This can be
* solved by adding a pre-removal callback to configfs, or passing
* around the cluster with the node. -jeffm
*/
Without the userspace heartbeat code I can't seem to create this situation and everything works nicely.
So I'm not yet in a position to really comment on
the .disconnect_notify changes.
--
Andrew Beekhof
"Would the last person to leave please turn out the enlightenment?" - TISM