fs/ceph/messenger: add separate lock for `out_queue`
PATCH 1/1 fs/ceph/messenger: add separate lock for `out_queue`
There is a lot of lock contention for ceph_connection.mutex because both the worker kthread (i.e. ceph_con_workfn()) and ceph_con_send() are using this mutex.
ceph_con_workfn() does the actual socket I/O and handles incoming packets; it can hold the mutex for very long periods of time. ceph_con_send(), on the other hand, may be called by arbitrary userspace processes invoking system calls for I/O on Ceph files. This means heavy contention of the mutex may slow down thousands of userspace processes.
However, ceph_con_send() accesses just con->out_queue and little else. The real work is later done by ceph_con_workfn(). It is therefore trivial to introduce a light-weight spinlock just for out_queue and eliminate almost all of the contention.
This is a /proc/lock_stat I captured on one of our web servers for 5 minutes without to this patch:
class name con-bounces contentions waittime-min waittime-max waittime-total waittime-avg acq-bounces acquisitions holdtime-min holdtime-max holdtime-total holdtime-avg
&con->mutex: 15918 15962 0.40 37877.04 7853358.52 492.00 13430455 33994666 0.08 93555.76 446452667.84 13.13
-----------
&con->mutex 15902 [<00000000aacd116f>] ceph_con_send+0x5c/0x170 &con->mutex 51 [<000000001b111558>] ceph_con_in_msg_alloc+0x74/0x1c0 &con->mutex 5 [<0000000068ae10b2>] ceph_con_keepalive+0x28/0x80 &con->mutex 3 [<000000009208fc49>] ceph_con_workfn+0x30/0x418 ----------- &con->mutex 13253 [<00000000c9c4e344>] ceph_con_process_message+0x90/0xd0 &con->mutex 1792 [<000000009208fc49>] ceph_con_workfn+0x30/0x418 &con->mutex 897 [<000000001b111558>] ceph_con_in_msg_alloc+0x74/0x1c0 &con->mutex 20 [<00000000aacd116f>] ceph_con_send+0x5c/0x170
ceph_con_process_message() is holding the mutex all the time, and there are lots of ceph_con_send() calls waiting for it. Each of these gets delayed by an average of 0.5 ms.
Now the same but with this patch:
class name con-bounces contentions waittime-min waittime-max waittime-total waittime-avg acq-bounces acquisitions holdtime-min holdtime-max holdtime-total holdtime-avg
&con->out_lock: 63452 64484 0.04 835.92 210874.20 3.27 13212464 36953693 0.04 902.60 16032508.44 0.43
--------------
&con->out_lock 42003 [<00000000e6da95fb>] ceph_con_send+0x64/0x1a0 &con->out_lock 4451 [<00000000981ab2f3>] ceph_con_discard_sent+0x48/0x188 &con->out_lock 18025 [<000000009ca1ffc1>] ceph_con_v1_try_write+0x2d8/0x768 &con->out_lock 5 [<0000000030330169>] ceph_con_get_out_msg+0xc0/0x210 -------------- &con->out_lock 34132 [<000000009ca1ffc1>] ceph_con_v1_try_write+0x2d8/0x768 &con->out_lock 23239 [<00000000e6da95fb>] ceph_con_send+0x64/0x1a0 &con->out_lock 7112 [<00000000981ab2f3>] ceph_con_discard_sent+0x48/0x188 &con->out_lock 1 [<0000000030330169>] ceph_con_get_out_msg+0xc0/0x210
&con->mutex: 16 16 7.96 7130.96 13680.60 855.04 4962257 28359208 0.08 237939.28 369261170.32 13.02
-----------
&con->mutex 16 [<00000000f3e4b849>] ceph_con_keepalive+0x28/0x80
-----------
&con->mutex 15 [<000000005fc149e2>] ceph_con_process_message+0x90/0xd0
&con->mutex 1 [<000000006b2ce95a>] ceph_con_in_msg_alloc+0x74/0x1c0Almost no contentions remain on the old mutex. The new out_lock has more contentions than before, but these are shorter: the total wait time of both is reduced by a factor of 35. The average wait time was reduced from 492 us to 3 us.
--- include/linux/ceph/messenger.h | 19 +++++++- net/ceph/messenger.c | 79 +++++++++++++++++++++++++++++----- net/ceph/messenger_v1.c | 17 ++++++++ net/ceph/messenger_v2.c | 31 +++++++++++++ 4 files changed, 135 insertions(+), 11 deletions(-)
include/linux/ceph/messenger.h +18 -1
@@ -476,7 +476,13 @@ struct ceph_connection {struct ceph_messenger *msgr; -int state;/* CEPH_CON_S_* */ +/** +* CEPH_CON_S_* +* +* Protected by both #mutex and #out_lock. Writing must hold +* both, reading either one. +*/ +int state;atomic_t sock_state;struct socket *sock;
@@ -490,6 +496,12 @@ struct ceph_connection {struct mutex mutex;/* out queue */ + +/** +* Protects #out_queue and #out_sent. +*/ +spinlock_t out_lock; +struct list_head out_queue;struct list_head out_sent;/* sending or sent but unacked */ u64 out_seq;/* last message queued for send */
@@ -550,6 +562,11 @@ void ceph_addr_set_port(struct ceph_entity_addr *addr, int p);void ceph_con_process_message (struct ceph_connection *con);int ceph_con_in_msg_alloc (struct ceph_connection *con,struct ceph_msg_header *hdr,int *skip); + +/** +* Caller must hold both `ceph_connection.mutex` and +* `ceph_connection.out_lock`. +*/ struct ceph_msg *ceph_con_get_out_msg (struct ceph_connection *con);/* messenger_v1.c */
net/ceph/messenger.c +69 -10
215 lines not on this page · raw message
net/ceph/messenger_v1.c +17 -0
@@ -888,7 +888,9 @@ static int process_connect(struct ceph_connection *con) }WARN_ON (con->state !=CEPH_CON_S_V1_CONNECT_MSG ); +spin_lock (&con->out_lock); con->state =CEPH_CON_S_OPEN ; +spin_unlock (&con->out_lock); con->v1.auth_retry =0 ;/* we authenticated; clear flag */ con->v1.peer_global_seq =le32_to_cpu (con->v1.in_reply.global_seq);
@@ -1343,7 +1345,9 @@ int ceph_con_v1_try_read(struct ceph_connection *con)if (ret <0 ) goto out; +spin_lock (&con->out_lock); con->state =CEPH_CON_S_V1_CONNECT_MSG ; +spin_unlock (&con->out_lock);/* * Received banner is good, exchange connection info.
@@ -1403,7 +1407,9 @@ int ceph_con_v1_try_read(struct ceph_connection *con)break ;case CEPH_MSGR_TAG_CLOSE :ceph_con_close_socket (con); +spin_lock (&con->out_lock); con->state =CEPH_CON_S_CLOSED ; +spin_unlock (&con->out_lock); goto out;default : goto bad_tag;
@@ -1481,7 +1487,9 @@ int ceph_con_v1_try_write(struct ceph_connection *con)/* open the socket first? */ if (con->state ==CEPH_CON_S_PREOPEN ) {BUG_ON (con->sock); +spin_lock (&con->out_lock); con->state =CEPH_CON_S_V1_BANNER ; +spin_unlock (&con->out_lock);con_out_kvec_reset (con);prepare_write_banner (con);
@@ -1536,18 +1544,26 @@ int ceph_con_v1_try_write(struct ceph_connection *con) } do_next: +/* hold con->out_lock while checking for things to write and +* unlock it after clearing CEPH_CON_F_WRITE_PENDING to avoid +* racing with ceph_con_send() +*/ +spin_lock (&con->out_lock);if (con->state ==CEPH_CON_S_OPEN ) {if (ceph_con_flag_test_and_clear (con,CEPH_CON_F_KEEPALIVE_PENDING )) { +spin_unlock (&con->out_lock);prepare_write_keepalive (con); goto more; }/* is anything else pending? */ if ((msg =ceph_con_get_out_msg (con)) !=NULL ) { +spin_unlock (&con->out_lock);prepare_write_message (con, msg); goto more; }if (con->in_seq > con->in_seq_acked) { +spin_unlock (&con->out_lock);prepare_write_ack (con); goto more; }
@@ -1555,6 +1571,7 @@ int ceph_con_v1_try_write(struct ceph_connection *con)/* Nothing to do! */ ceph_con_flag_clear (con,CEPH_CON_F_WRITE_PENDING ); +spin_unlock (&con->out_lock);dout ("try_write nothing else to write.\n" ); ret =0 ; out:
net/ceph/messenger_v2.c +31 -0
@@ -1744,7 +1744,9 @@ static int prepare_read_banner_prefix(struct ceph_connection *con)reset_in_kvecs (con);add_in_kvec (con, buf,CEPH_BANNER_V2_PREFIX_LEN );add_in_sign_kvec (con, buf,CEPH_BANNER_V2_PREFIX_LEN ); +spin_lock (&con->out_lock); con->state =CEPH_CON_S_V2_BANNER_PREFIX ; +spin_unlock (&con->out_lock);return 0 ; }
@@ -1760,7 +1762,9 @@ static int prepare_read_banner_payload(struct ceph_connection *con,reset_in_kvecs (con);add_in_kvec (con, buf, payload_len);add_in_sign_kvec (con, buf, payload_len); +spin_lock (&con->out_lock); con->state =CEPH_CON_S_V2_BANNER_PAYLOAD ; +spin_unlock (&con->out_lock);return 0 ; }
@@ -2181,7 +2185,9 @@ static int process_banner_payload(struct ceph_connection *con)return ret; } +spin_lock (&con->out_lock); con->state =CEPH_CON_S_V2_HELLO ; +spin_unlock (&con->out_lock);prepare_read_preamble (con);return 0 ;
@@ -2249,7 +2255,9 @@ static int process_hello(struct ceph_connection *con, void *p, void *end)return ret; } +spin_lock (&con->out_lock); con->state =CEPH_CON_S_V2_AUTH ; +spin_unlock (&con->out_lock);return 0 ; bad:
@@ -2409,7 +2417,9 @@ static int process_auth_done(struct ceph_connection *con, void *p, void *end) goto out; } +spin_lock (&con->out_lock); con->state =CEPH_CON_S_V2_AUTH_SIGNATURE ; +spin_unlock (&con->out_lock); out:memzero_explicit (session_key,sizeof (session_key));
@@ -2451,7 +2461,9 @@ static int process_auth_signature(struct ceph_connection *con,return ret; } +spin_lock (&con->out_lock); con->state =CEPH_CON_S_V2_SESSION_CONNECT ; +spin_unlock (&con->out_lock); }else { ret =prepare_session_reconnect (con);if (ret) {
@@ -2459,7 +2471,9 @@ static int process_auth_signature(struct ceph_connection *con,return ret; } +spin_lock (&con->out_lock); con->state =CEPH_CON_S_V2_SESSION_RECONNECT ; +spin_unlock (&con->out_lock); }return 0 ;
@@ -2544,7 +2558,9 @@ static int process_server_ident(struct ceph_connection *con,free_conn_bufs (con); con->delay =0 ;/* reset backoff memory */ +spin_lock (&con->out_lock); con->state =CEPH_CON_S_OPEN ; +spin_unlock (&con->out_lock); con->v2.out_state =OUT_S_GET_NEXT ;return 0 ;
@@ -2595,7 +2611,9 @@ static int process_session_reconnect_ok(struct ceph_connection *con,free_conn_bufs (con); con->delay =0 ;/* reset backoff memory */ +spin_lock (&con->out_lock); con->state =CEPH_CON_S_OPEN ; +spin_unlock (&con->out_lock); con->v2.out_state =OUT_S_GET_NEXT ;return 0 ;
@@ -2710,7 +2728,9 @@ static int process_session_reset(struct ceph_connection *con,return ret; } +spin_lock (&con->out_lock); con->state =CEPH_CON_S_V2_SESSION_CONNECT ; +spin_unlock (&con->out_lock);return 0 ; bad:
@@ -3285,6 +3305,7 @@ static int populate_out_iter(struct ceph_connection *con)if (con->state !=CEPH_CON_S_OPEN ) {WARN_ON (con->state <CEPH_CON_S_V2_BANNER_PREFIX || con->state >CEPH_CON_S_V2_SESSION_RECONNECT ); +spin_lock (&con->out_lock); goto nothing_pending; }
@@ -3315,19 +3336,28 @@ static int populate_out_iter(struct ceph_connection *con) }WARN_ON (con->v2.out_state !=OUT_S_GET_NEXT ); + +/* hold con->out_lock while checking for things to write and +* unlock it after clearing CEPH_CON_F_WRITE_PENDING to avoid +* racing with ceph_con_send() +*/ +spin_lock (&con->out_lock);if (ceph_con_flag_test_and_clear (con,CEPH_CON_F_KEEPALIVE_PENDING )) { +spin_unlock (&con->out_lock); ret =prepare_keepalive2 (con);if (ret) {pr_err ("prepare_keepalive2 failed: %d\n" , ret);return ret; } }else if ((msg =ceph_con_get_out_msg (con)) !=NULL ) { +spin_unlock (&con->out_lock); ret =prepare_message (con, msg);if (ret) {pr_err ("prepare_message failed: %d\n" , ret);return ret; } }else if (con->in_seq > con->in_seq_acked) { +spin_unlock (&con->out_lock); ret =prepare_ack (con);if (ret) {pr_err ("prepare_ack failed: %d\n" , ret);
@@ -3348,6 +3378,7 @@ static int populate_out_iter(struct ceph_connection *con)WARN_ON (iov_iter_count (&con->v2.out_iter));dout ("%s con %p nothing pending\n" , __func__, con);ceph_con_flag_clear (con,CEPH_CON_F_WRITE_PENDING ); +spin_unlock (&con->out_lock);return 0 ; }
Signature · 1 line
2.47.3
Hi Max,
Thanks for the patch.
Signature · 1 line
Alex Markuze