dlm: if forced set no linger before close
Alexander Aring
maintainerPATCH 1/1 dlm: if forced set no linger before close
I currently observe an issue where an ETIMEDOUT occurred during a connect after the same node got fenced before. On the peer side netstat showed still the previous connection in FIN_WAIT1 state and having something in the send socket queue. I assume the previous fence did a close by calling dlm_lowcomms_close() but still had something in the send queue, so it ended up by doing a graceful shutdown but in this case we know it cannot succeed as the peer is already gone and lost its transport layer state. When the fenced node came back and it wants to connect it returned ETIMEDOUT as the peer it wants to connect still maintained the previous connection that sits in FIN_WAIT1.
We will call sock_no_linger() if the socket close came from dlm_lowcomms_close() that is usually called when the cluster manager removes a node from the cluster e.g. on fencing, all other non hard closed cases I left the behaviour as it was. This should prevent of trying a graceful shutdown when something sits in the socket send queue and discard all messages. In case of a fence this is what we want as DLM recovery will handle a fenced node.
--- fs/dlm/lowcomms.c | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-)
fs/dlm/lowcomms.c +28 -9
@@ -753,12 +753,13 @@ static void stop_connection_io(struct connection *con) }/* Close a remote connection and tidy up */ -static void close_connection (struct connection *con,bool and_other) +static void close_connection (struct connection *con,bool and_other, +bool no_linger) {struct writequeue_entry *e;if (con->othercon && and_other) -close_connection (con->othercon, false); +close_connection (con->othercon, false, no_linger);down_write (&con->sock_lock);if (!con->sock) {
@@ -766,6 +767,24 @@ static void close_connection(struct connection *con, bool and_other)return ; } +/* in cases e.g. of fencing we know the other peer is fenced +* out of the cluster. We assume that in this case the peers +* transport layer state was gone. By default linger is off +* that means the kernel will in background try to make a +* graceful shutdown. We know that this is not possible and +* a fenced node coming back and join a lockspace it could +* run into ETIMEDOUT as a previous connection is still +* maintained until some timeout occurs and the graceful +* shutdown was given up. This timeout can vary regarding to +* different parameters. To avoid the graceful shutdown we +* enable linger and set the linger timeout handling to 0, this +* will do a hard close without trying to graceful shutdown +* the socket handling and immediately close the connection +* with discard any data in the socket queues. +*/ +if (no_linger) +sock_no_linger (con->sock->sk); +dlm_close_sock (&con->sock);/* if we send a writequeue entry only a half way, we drop the
@@ -830,7 +849,7 @@ static void shutdown_connection(struct connection *con, bool and_other)return ; force_close: -close_connection (con, false); +close_connection (con, false, true); } static struct processqueue_entry *new_processqueue_entry(int nodeid,
@@ -1067,7 +1086,7 @@ static int accept_from_sock(void)set_bit (CF_IS_OTHERCON , &othercon->flags); }else {/* close other sock con if we have something new */ -close_connection (othercon, false); +close_connection (othercon, false, false); }down_write (&othercon->sock_lock);
@@ -1457,7 +1476,7 @@ int dlm_lowcomms_close(int nodeid)stop_connection_io (con);log_print ("io handling for node: %d stopped" , nodeid); -close_connection (con, true); +close_connection (con, true, true);spin_lock (&connections_lock);hlist_del_rcu (&con->list);
@@ -1503,7 +1522,7 @@ static void process_recv_sockets(struct work_struct *work)/* CF_RECV_PENDING cleared */ break ;case DLM_IO_EOF : -close_connection (con, false); +close_connection (con, false, false);wake_up (&con->shutdown_wait);/* CF_RECV_PENDING cleared */ break ;
@@ -1531,7 +1550,7 @@ static void process_recv_sockets(struct work_struct *work)default :if (ret <0 ) {if (test_bit (CF_IS_OTHERCON , &con->flags)) { -close_connection (con, false); +close_connection (con, false, false); }else {spin_lock_bh (&con->writequeue_lock);lowcomms_queue_swork (con);
@@ -1667,7 +1686,7 @@ static void process_send_sockets(struct work_struct *work)break ;default :if (ret <0 ) { -close_connection (con, false); +close_connection (con, false, false);/* CF_SEND_PENDING cleared */ spin_lock_bh (&con->writequeue_lock);
@@ -1733,7 +1752,7 @@ void dlm_lowcomms_shutdown(void)shutdown_connection (con, true);stop_connection_io (con);flush_workqueue (process_workqueue); -close_connection (con, true); +close_connection (con, true, false);clean_one_writequeue (con);if (con->othercon)
Signature · 1 line
2.43.0