ksmbd: smbd: call rdma_accept() under CM handler
Subsystems (2)
PATCH 1/1 ksmbd: smbd: call rdma_accept() under CM handler
if CONFIG_LOCKDEP is enabled, the following kernel warning message is generated because rdma_accept() is called not under CM(Connection Manager) handler.
[ 63.211405 ] WARNING: CPU: 1 PID: 345 at drivers/infiniband/core/cma.c:4405 rdma_accept+0x17a/0x350
[ 63.212080 ] RIP: 0010:rdma_accept+0x17a/0x350
...
[ 63.214036 ] Call Trace: [ 63.214098 ] <TASK> [ 63.214185 ] smb_direct_accept_client+0xb4/0x170 [ksmbd] [ 63.214412 ] smb_direct_prepare+0x322/0x8c0 [ksmbd] [ 63.214555 ] ? rcu_read_lock_sched_held+0x3a/0x70 [ 63.214700 ] ksmbd_conn_handler_loop+0x63/0x270 [ksmbd] [ 63.214826 ] ? ksmbd_conn_alive+0x80/0x80 [ksmbd] [ 63.214952 ] kthread+0x171/0x1a0 [ 63.215039 ] ? set_kthread_struct+0x40/0x40 [ 63.215128 ] ret_from_fork+0x22/0x30
To avoid this, move creating a queue pair and accepting a client from transport_ops->prepare() to smb_direct_handle_connect_request().
--- fs/ksmbd/transport_rdma.c | 97 +++++++++++++++++++++++---------------- 1 file changed, 57 insertions(+), 40 deletions(-)
fs/ksmbd/transport_rdma.c +57 -40
@@ -568,6 +568,7 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) } t->negotiation_requested = true; t->full_packet_received = true; +enqueue_reassembly (t, recvmsg,0 );wake_up_interruptible (&t->wait_status);break ;case SMB_DIRECT_MSG_DATA_TRANSFER : {
@@ -1594,19 +1595,13 @@ static int smb_direct_accept_client(struct smb_direct_transport *t)pr_err ("error at rdma_accept: %d\n" , ret);return ret; } - -wait_event_interruptible (t->wait_status, - t->status !=SMB_DIRECT_CS_NEW ); -if (t->status !=SMB_DIRECT_CS_CONNECTED ) -return -ENOTCONN ;return 0 ; } -static int smb_direct_negotiate (struct smb_direct_transport *t) +static int smb_direct_prepare_negotiation (struct smb_direct_transport *t) {int ret;struct smb_direct_recvmsg *recvmsg; -struct smb_direct_negotiate_req *req; recvmsg =get_free_recvmsg (t);if (!recvmsg)
@@ -1616,44 +1611,20 @@ static int smb_direct_negotiate(struct smb_direct_transport *t) ret =smb_direct_post_recv (t, recvmsg);if (ret) {pr_err ("Can't post recv: %d\n" , ret); - goto out; + goto out_err; } t->negotiation_requested = false; ret =smb_direct_accept_client (t);if (ret) {pr_err ("Can't accept client\n" ); - goto out; + goto out_err; }smb_direct_post_recv_credits (&t->post_recv_credits_work.work); - -ksmbd_debug (RDMA ,"Waiting for SMB_DIRECT negotiate request\n" ); - ret =wait_event_interruptible_timeout (t->wait_status, - t->negotiation_requested || - t->status ==SMB_DIRECT_CS_DISCONNECTED , -SMB_DIRECT_NEGOTIATE_TIMEOUT *HZ ); -if (ret <=0 || t->status ==SMB_DIRECT_CS_DISCONNECTED ) { - ret = ret <0 ? ret : -ETIMEDOUT ; - goto out; - } - - ret =smb_direct_check_recvmsg (recvmsg); -if (ret == -ECONNABORTED ) - goto out; - - req = (struct smb_direct_negotiate_req *)recvmsg->packet; - t->max_recv_size =min_t (int, t->max_recv_size, -le32_to_cpu (req->preferred_send_size)); - t->max_send_size =min_t (int, t->max_send_size, -le32_to_cpu (req->max_receive_size)); - t->max_fragmented_send_size = -le32_to_cpu (req->max_fragmented_size); - - ret =smb_direct_send_negotiate_response (t, ret); -out: -if (recvmsg) -put_recvmsg (t, recvmsg); +return 0 ; +out_err: +put_recvmsg (t, recvmsg);return ret; }
@@ -1890,6 +1861,47 @@ static int smb_direct_create_qpair(struct smb_direct_transport *t,static int smb_direct_prepare (struct ksmbd_transport *t) {struct smb_direct_transport *st =smb_trans_direct_transfort (t); +struct smb_direct_recvmsg *recvmsg; +struct smb_direct_negotiate_req *req; +int ret; + +ksmbd_debug (RDMA ,"Waiting for SMB_DIRECT negotiate request\n" ); + ret =wait_event_interruptible_timeout (st->wait_status, + st->negotiation_requested || + st->status ==SMB_DIRECT_CS_DISCONNECTED , +SMB_DIRECT_NEGOTIATE_TIMEOUT *HZ ); +if (ret <=0 || st->status ==SMB_DIRECT_CS_DISCONNECTED ) +return ret <0 ? ret : -ETIMEDOUT ; + + recvmsg =get_first_reassembly (st); +if (!recvmsg) +return -ECONNABORTED ; + + ret =smb_direct_check_recvmsg (recvmsg); +if (ret == -ECONNABORTED ) + goto out; + + req = (struct smb_direct_negotiate_req *)recvmsg->packet; + st->max_recv_size =min_t (int, st->max_recv_size, +le32_to_cpu (req->preferred_send_size)); + st->max_send_size =min_t (int, st->max_send_size, +le32_to_cpu (req->max_receive_size)); + st->max_fragmented_send_size = +le32_to_cpu (req->max_fragmented_size); + + ret =smb_direct_send_negotiate_response (st, ret); +out: +spin_lock_irq (&st->reassembly_queue_lock); + st->reassembly_queue_length--; +list_del (&recvmsg->list); +spin_unlock_irq (&st->reassembly_queue_lock); +put_recvmsg (st, recvmsg); + +return ret; +} + +static int smb_direct_connect (struct smb_direct_transport *st) +{int ret;struct ib_qp_cap qp_cap;
@@ -1911,13 +1923,11 @@ static int smb_direct_prepare(struct ksmbd_transport *t)return ret; } - ret =smb_direct_negotiate (st); + ret =smb_direct_prepare_negotiation (st);if (ret) {pr_err ("Can't negotiate: %d\n" , ret);return ret; } - - st->status =SMB_DIRECT_CS_CONNECTED ;return 0 ; }
@@ -1933,6 +1943,7 @@ static bool rdma_frwr_is_supported(struct ib_device_attr *attrs)static int smb_direct_handle_connect_request (struct rdma_cm_id *new_cm_id) {struct smb_direct_transport *t; +int ret;if (!rdma_frwr_is_supported (&new_cm_id->device->attrs)) { ksmbd_debug(RDMA,
@@ -1945,11 +1956,17 @@ static int smb_direct_handle_connect_request(struct rdma_cm_id *new_cm_id)if (!t)return -ENOMEM ; + ret =smb_direct_connect (t); +if (ret) { +free_transport (t); +return ret; + } +KSMBD_TRANS (t)->handler =kthread_run (ksmbd_conn_handler_loop,KSMBD_TRANS (t)->conn,"ksmbd:r%u" , smb_direct_port);if (IS_ERR (KSMBD_TRANS (t)->handler)) { -int ret =PTR_ERR (KSMBD_TRANS (t)->handler); + ret =PTR_ERR (KSMBD_TRANS (t)->handler);pr_err ("Can't start thread\n" );free_transport (t);
Signature · 1 line
2.25.1
Namjae Jeon
maintainer2022-01-03 9:02 GMT+09:00, Hyunchul Lee <[email protected]>:
Quoted 9 lines
if
CONFIG_LOCKDEPis enabled, the following kernel warning message is generated becauserdma_accept()is called not under CM(Connection Manager) handler.[ 63.211405 ] WARNING: CPU: 1 PID: 345 at
drivers/infiniband/core/cma.c:4405rdma_accept+0x17a/0x350
[ 63.212080 ] RIP: 0010:rdma_accept+0x17a/0x350
...
Quoted 10 lines
[ 63.214036 ] Call Trace: [ 63.214098 ] <TASK> [ 63.214185 ] smb_direct_accept_client+0xb4/0x170 [ksmbd] [ 63.214412 ] smb_direct_prepare+0x322/0x8c0 [ksmbd] [ 63.214555 ] ? rcu_read_lock_sched_held+0x3a/0x70 [ 63.214700 ] ksmbd_conn_handler_loop+0x63/0x270 [ksmbd] [ 63.214826 ] ? ksmbd_conn_alive+0x80/0x80 [ksmbd] [ 63.214952 ] kthread+0x171/0x1a0 [ 63.215039 ] ? set_kthread_struct+0x40/0x40 [ 63.215128 ] ret_from_fork+0x22/0x30
I could not understand why lockdep trigger this warning. Could you elaborate more ?
Quoted 15 lines
To avoid this, move creating a queue pair and accepting a client from
transport_ops->prepare()tosmb_direct_handle_connect_request().Signed-off-by Hyunchul Lee <[email protected]>--- fs/ksmbd/transport_rdma.c | 97 +++++++++++++++++++++++---------------- 1 file changed, 57 insertions(+), 40 deletions(-)fs/ksmbd/transport_rdma.c +0
-0
fs/ksmbd/transport_rdma.c static void recv_done(struct ib_cq *cq, struct ib_wc · line 572
Quoted 5 lines
fs/ksmbd/transport_rdma.c +1
-0@@ -568,6 +568,7 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) } t->negotiation_requested = true; t->full_packet_received = true; +enqueue_reassembly (t, recvmsg,0 );
Is this a fix related to this patch?
Quoted 151 lines
fs/ksmbd/transport_rdma.c +0
-0wake_up_interruptible (&t->wait_status);break ;fs/ksmbd/transport_rdma.c +1
-7case SMB_DIRECT_MSG_DATA_TRANSFER : {@@ -1594,19 +1595,13 @@ static int smb_direct_accept_client(struct smb_direct_transport *t) pr_err("error at rdma_accept: %d\n", ret);return ret; } - -wait_event_interruptible (t->wait_status, - t->status !=SMB_DIRECT_CS_NEW ); -if (t->status !=SMB_DIRECT_CS_CONNECTED ) -return -ENOTCONN ;return 0 ; } -static int smb_direct_negotiate (struct smb_direct_transport *t) +static int smb_direct_prepare_negotiation (struct smb_direct_transport *t) {int ret;struct smb_direct_recvmsg *recvmsg; -struct smb_direct_negotiate_req *req; recvmsg =get_free_recvmsg (t);fs/ksmbd/transport_rdma.c +48
-32if (!recvmsg)@@ -1616,44 +1611,20 @@ static int smb_direct_negotiate(struct smb_direct_transport *t) ret =smb_direct_post_recv (t, recvmsg);if (ret) {pr_err ("Can't post recv: %d\n" , ret); - goto out; + goto out_err; } t->negotiation_requested = false; ret =smb_direct_accept_client (t);if (ret) {pr_err ("Can't accept client\n" ); - goto out; + goto out_err; }smb_direct_post_recv_credits (&t->post_recv_credits_work.work); - -ksmbd_debug (RDMA ,"Waiting for SMB_DIRECT negotiate request\n" ); - ret =wait_event_interruptible_timeout (t->wait_status, - t->negotiation_requested || - t->status ==SMB_DIRECT_CS_DISCONNECTED , -SMB_DIRECT_NEGOTIATE_TIMEOUT *HZ ); -if (ret <=0 || t->status ==SMB_DIRECT_CS_DISCONNECTED ) { - ret = ret <0 ? ret : -ETIMEDOUT ; - goto out; - } - - ret =smb_direct_check_recvmsg (recvmsg); -if (ret == -ECONNABORTED ) - goto out; - - req = (struct smb_direct_negotiate_req *)recvmsg->packet; - t->max_recv_size =min_t (int, t->max_recv_size, -le32_to_cpu (req->preferred_send_size)); - t->max_send_size =min_t (int, t->max_send_size, -le32_to_cpu (req->max_receive_size)); - t->max_fragmented_send_size = -le32_to_cpu (req->max_fragmented_size); - - ret =smb_direct_send_negotiate_response (t, ret); -out: -if (recvmsg) -put_recvmsg (t, recvmsg); +return 0 ; +out_err: +put_recvmsg (t, recvmsg);return ret; }@@ -1890,6 +1861,47 @@ static int smb_direct_create_qpair(structsmb_direct_transport *t, static int smb_direct_prepare(struct ksmbd_transport *t) { struct smb_direct_transport *st = smb_trans_direct_transfort(t); +struct smb_direct_recvmsg *recvmsg; +struct smb_direct_negotiate_req *req; +int ret; + +ksmbd_debug (RDMA ,"Waiting for SMB_DIRECT negotiate request\n" ); + ret =wait_event_interruptible_timeout (st->wait_status, + st->negotiation_requested || + st->status ==SMB_DIRECT_CS_DISCONNECTED , +SMB_DIRECT_NEGOTIATE_TIMEOUT *HZ ); +if (ret <=0 || st->status ==SMB_DIRECT_CS_DISCONNECTED ) +return ret <0 ? ret : -ETIMEDOUT ; + + recvmsg =get_first_reassembly (st); +if (!recvmsg) +return -ECONNABORTED ; + + ret =smb_direct_check_recvmsg (recvmsg); +if (ret == -ECONNABORTED ) + goto out; + + req = (struct smb_direct_negotiate_req *)recvmsg->packet; + st->max_recv_size =min_t (int, st->max_recv_size, +le32_to_cpu (req->preferred_send_size)); + st->max_send_size =min_t (int, st->max_send_size, +le32_to_cpu (req->max_receive_size)); + st->max_fragmented_send_size = +le32_to_cpu (req->max_fragmented_size); + + ret =smb_direct_send_negotiate_response (st, ret); +out: +spin_lock_irq (&st->reassembly_queue_lock); + st->reassembly_queue_length--; +list_del (&recvmsg->list); +spin_unlock_irq (&st->reassembly_queue_lock); +put_recvmsg (st, recvmsg); + +return ret; +} + +static int smb_direct_connect (struct smb_direct_transport *st) +{int ret;struct ib_qp_cap qp_cap;@@ -1911,13 +1923,11 @@ static int smb_direct_prepare(struct ksmbd_transport *t)return ret; } - ret =smb_direct_negotiate (st); + ret =smb_direct_prepare_negotiation (st);if (ret) {pr_err ("Can't negotiate: %d\n" , ret);return ret; } - - st->status =SMB_DIRECT_CS_CONNECTED ;return 0 ; }@@ -1933,6 +1943,7 @@ static bool rdma_frwr_is_supported(struct ib_device_attr *attrs)static int smb_direct_handle_connect_request (struct rdma_cm_id *new_cm_id) {struct smb_direct_transport *t; +int ret;if (!rdma_frwr_is_supported (&new_cm_id->device->attrs)) {fs/ksmbd/transport_rdma.c +0
-0ksmbd_debug(RDMA,
fs/ksmbd/transport_rdma.c static int smb_direct_handle_connect_request(struct · line 1963
Quoted 8 lines
fs/ksmbd/transport_rdma.c +4
-0@@ -1945,11 +1956,17 @@ static int smb_direct_handle_connect_request(struct rdma_cm_id *new_cm_id)if (!t)return -ENOMEM ; + ret =smb_direct_connect (t); +if (ret) { +free_transport (t); +return ret;
I think that it is better to use goto statement to out after freeing transport structure.
Quoted 15 lines
fs/ksmbd/transport_rdma.c +3
-1+ } +KSMBD_TRANS (t)->handler =kthread_run (ksmbd_conn_handler_loop,KSMBD_TRANS (t)->conn,"ksmbd:r%u" , smb_direct_port);if (IS_ERR (KSMBD_TRANS (t)->handler)) { -int ret =PTR_ERR (KSMBD_TRANS (t)->handler); + ret =PTR_ERR (KSMBD_TRANS (t)->handler);pr_err ("Can't start thread\n" );
free_transport(t);
--
2.25.1
2022년 1월 4일 (화) 오전 9:45, Namjae Jeon <[email protected]>님이 작성:
Quoted 21 lines
2022-01-03 9:02 GMT+09:00, Hyunchul Lee <[email protected]>:
Quoted 9 lines
if
CONFIG_LOCKDEPis enabled, the following kernel warning message is generated becauserdma_accept()is called not under CM(Connection Manager) handler.[ 63.211405 ] WARNING: CPU: 1 PID: 345 at
drivers/infiniband/core/cma.c:4405rdma_accept+0x17a/0x350
[ 63.212080 ] RIP: 0010:rdma_accept+0x17a/0x350
...Quoted 10 lines
[ 63.214036 ] Call Trace: [ 63.214098 ] <TASK> [ 63.214185 ] smb_direct_accept_client+0xb4/0x170 [ksmbd] [ 63.214412 ] smb_direct_prepare+0x322/0x8c0 [ksmbd] [ 63.214555 ] ? rcu_read_lock_sched_held+0x3a/0x70 [ 63.214700 ] ksmbd_conn_handler_loop+0x63/0x270 [ksmbd] [ 63.214826 ] ? ksmbd_conn_alive+0x80/0x80 [ksmbd] [ 63.214952 ] kthread+0x171/0x1a0 [ 63.215039 ] ? set_kthread_struct+0x40/0x40 [ 63.215128 ] ret_from_fork+0x22/0x30
Quoted 3 lines
I could not understand why lockdep trigger this warning. Could you elaborate more ?
rdma_accept checks whether the handler_mutex is held. if not, this warning is generated. And CM holds the handler_mutex before ksmbd's handler callback is called.
Quoted 10 lines
Quoted 10 lines
To avoid this, move creating a queue pair and accepting a client from
transport_ops->prepare()tosmb_direct_handle_connect_request().Signed-off-by Hyunchul Lee <[email protected]>--- fs/ksmbd/transport_rdma.c | 97 +++++++++++++++++++++++---------------- 1 file changed, 57 insertions(+), 40 deletions(-)
Quoted 11 lines
fs/ksmbd/transport_rdma.c static void recv_done(struct ib_cq *cq, struct ib_wc · line 572
Quoted 10 lines
fs/ksmbd/transport_rdma.c +1
-0@@ -568,6 +568,7 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) } t->negotiation_requested = true; t->full_packet_received = true; +enqueue_reassembly (t, recvmsg,0 );Is this a fix related to this patch?
Yes, this is required to receive the negotiation request from a client.
Before this patch, posting a buffer and waiting for the request is done in a function. Because
we have the reference of the buffer, we don't need to append it into the queue.
Quoted 155 lines
Quoted 155 lines
fs/ksmbd/transport_rdma.c +0
-0wake_up_interruptible (&t->wait_status);break ;case SMB_DIRECT_MSG_DATA_TRANSFER: {fs/ksmbd/transport_rdma.c +1
-7@@ -1594,19 +1595,13 @@ static int smb_direct_accept_client(struct smb_direct_transport *t) pr_err("error at rdma_accept: %d\n", ret);return ret; } - -wait_event_interruptible (t->wait_status, - t->status !=SMB_DIRECT_CS_NEW ); -if (t->status !=SMB_DIRECT_CS_CONNECTED ) -return -ENOTCONN ;return 0 ; } -static int smb_direct_negotiate (struct smb_direct_transport *t) +static int smb_direct_prepare_negotiation (struct smb_direct_transport *t) {int ret;struct smb_direct_recvmsg *recvmsg; -struct smb_direct_negotiate_req *req; recvmsg =get_free_recvmsg (t);if (!recvmsg)
fs/ksmbd/transport_rdma.c +48
-32@@ -1616,44 +1611,20 @@ static int smb_direct_negotiate(struct smb_direct_transport *t) ret =smb_direct_post_recv (t, recvmsg);if (ret) {pr_err ("Can't post recv: %d\n" , ret); - goto out; + goto out_err; } t->negotiation_requested = false; ret =smb_direct_accept_client (t);if (ret) {pr_err ("Can't accept client\n" ); - goto out; + goto out_err; }smb_direct_post_recv_credits (&t->post_recv_credits_work.work); - -ksmbd_debug (RDMA ,"Waiting for SMB_DIRECT negotiate request\n" ); - ret =wait_event_interruptible_timeout (t->wait_status, - t->negotiation_requested || - t->status ==SMB_DIRECT_CS_DISCONNECTED , -SMB_DIRECT_NEGOTIATE_TIMEOUT *HZ ); -if (ret <=0 || t->status ==SMB_DIRECT_CS_DISCONNECTED ) { - ret = ret <0 ? ret : -ETIMEDOUT ; - goto out; - } - - ret =smb_direct_check_recvmsg (recvmsg); -if (ret == -ECONNABORTED ) - goto out; - - req = (struct smb_direct_negotiate_req *)recvmsg->packet; - t->max_recv_size =min_t (int, t->max_recv_size, -le32_to_cpu (req->preferred_send_size)); - t->max_send_size =min_t (int, t->max_send_size, -le32_to_cpu (req->max_receive_size)); - t->max_fragmented_send_size = -le32_to_cpu (req->max_fragmented_size); - - ret =smb_direct_send_negotiate_response (t, ret); -out: -if (recvmsg) -put_recvmsg (t, recvmsg); +return 0 ; +out_err: +put_recvmsg (t, recvmsg);return ret; }@@ -1890,6 +1861,47 @@ static int smb_direct_create_qpair(structsmb_direct_transport *t, static int smb_direct_prepare(struct ksmbd_transport *t) { struct smb_direct_transport *st = smb_trans_direct_transfort(t); +struct smb_direct_recvmsg *recvmsg; +struct smb_direct_negotiate_req *req; +int ret; + +ksmbd_debug (RDMA ,"Waiting for SMB_DIRECT negotiate request\n" ); + ret =wait_event_interruptible_timeout (st->wait_status, + st->negotiation_requested || + st->status ==SMB_DIRECT_CS_DISCONNECTED , +SMB_DIRECT_NEGOTIATE_TIMEOUT *HZ ); +if (ret <=0 || st->status ==SMB_DIRECT_CS_DISCONNECTED ) +return ret <0 ? ret : -ETIMEDOUT ; + + recvmsg =get_first_reassembly (st); +if (!recvmsg) +return -ECONNABORTED ; + + ret =smb_direct_check_recvmsg (recvmsg); +if (ret == -ECONNABORTED ) + goto out; + + req = (struct smb_direct_negotiate_req *)recvmsg->packet; + st->max_recv_size =min_t (int, st->max_recv_size, +le32_to_cpu (req->preferred_send_size)); + st->max_send_size =min_t (int, st->max_send_size, +le32_to_cpu (req->max_receive_size)); + st->max_fragmented_send_size = +le32_to_cpu (req->max_fragmented_size); + + ret =smb_direct_send_negotiate_response (st, ret); +out: +spin_lock_irq (&st->reassembly_queue_lock); + st->reassembly_queue_length--; +list_del (&recvmsg->list); +spin_unlock_irq (&st->reassembly_queue_lock); +put_recvmsg (st, recvmsg); + +return ret; +} + +static int smb_direct_connect (struct smb_direct_transport *st) +{int ret;struct ib_qp_cap qp_cap;@@ -1911,13 +1923,11 @@ static int smb_direct_prepare(struct ksmbd_transport *t)return ret; } - ret =smb_direct_negotiate (st); + ret =smb_direct_prepare_negotiation (st);if (ret) {pr_err ("Can't negotiate: %d\n" , ret);return ret; } - - st->status =SMB_DIRECT_CS_CONNECTED ;return 0 ; }@@ -1933,6 +1943,7 @@ static bool rdma_frwr_is_supported(struct ib_device_attr *attrs)static int smb_direct_handle_connect_request (struct rdma_cm_id *new_cm_id) {struct smb_direct_transport *t; +int ret;if (!rdma_frwr_is_supported (&new_cm_id->device->attrs)) {ksmbd_debug(RDMA,
fs/ksmbd/transport_rdma.c +0
-0@@ -1945,11 +1956,17 @@ static int smb_direct_handle_connect_request(struct rdma_cm_id *new_cm_id)if (!t)return -ENOMEM ;
Quoted 6 lines
fs/ksmbd/transport_rdma.c static int smb_direct_handle_connect_request(struct rdma_cm_id *new_cm_id) · line 1962
Quoted 4 lines
fs/ksmbd/transport_rdma.c +4
-0+ ret =smb_direct_connect (t); +if (ret) { +free_transport (t); +return ret;I think that it is better to use goto statement to out after freeing transport structure.
Okay, I will change it.
Quoted 15 lines
Quoted 15 lines
fs/ksmbd/transport_rdma.c +3
-1+ } +KSMBD_TRANS (t)->handler =kthread_run (ksmbd_conn_handler_loop,KSMBD_TRANS (t)->conn,"ksmbd:r%u" , smb_direct_port);if (IS_ERR (KSMBD_TRANS (t)->handler)) { -int ret =PTR_ERR (KSMBD_TRANS (t)->handler); + ret =PTR_ERR (KSMBD_TRANS (t)->handler);pr_err ("Can't start thread\n" );free_transport(t);--
2.25.1
Signature · 2 lines
Thanks, Hyunchul
Namjae Jeon
maintainer2022-01-04 10:10 GMT+09:00, Hyunchul Lee <[email protected]>:
Quoted 22 lines
2022년 1월 4일 (화) 오전 9:45, Namjae Jeon <[email protected]>님이 작성:
Quoted 21 lines
2022-01-03 9:02 GMT+09:00, Hyunchul Lee <[email protected]>:
Quoted 19 lines
if
CONFIG_LOCKDEPis enabled, the following kernel warning message is generated becauserdma_accept()is called not under CM(Connection Manager) handler.[ 63.211405 ] WARNING: CPU: 1 PID: 345 at
drivers/infiniband/core/cma.c:4405rdma_accept+0x17a/0x350
[ 63.212080 ] RIP: 0010:rdma_accept+0x17a/0x350
...[ 63.214036 ] Call Trace: [ 63.214098 ] <TASK> [ 63.214185 ] smb_direct_accept_client+0xb4/0x170 [ksmbd] [ 63.214412 ] smb_direct_prepare+0x322/0x8c0 [ksmbd] [ 63.214555 ] ? rcu_read_lock_sched_held+0x3a/0x70 [ 63.214700 ] ksmbd_conn_handler_loop+0x63/0x270 [ksmbd] [ 63.214826 ] ? ksmbd_conn_alive+0x80/0x80 [ksmbd] [ 63.214952 ] kthread+0x171/0x1a0 [ 63.215039 ] ? set_kthread_struct+0x40/0x40 [ 63.215128 ] ret_from_fork+0x22/0x30
Quoted 7 lines
Quoted 3 lines
I could not understand why lockdep trigger this warning. Could you elaborate more ?
rdma_acceptchecks whether thehandler_mutexis held. if not, this warning is generated. And CM holds thehandler_mutexbefore ksmbd's handler callback is called.
Okay, please update patch description with this.
Quoted 24 lines
Quoted 11 lines
Quoted 11 lines
To avoid this, move creating a queue pair and accepting a client from
transport_ops->prepare()tosmb_direct_handle_connect_request().Signed-off-by Hyunchul Lee <[email protected]>--- fs/ksmbd/transport_rdma.c | 97 +++++++++++++++++++++++---------------- 1 file changed, 57 insertions(+), 40 deletions(-)Quoted 12 lines
Quoted 11 lines
fs/ksmbd/transport_rdma.c +0
-0@@ -568,6 +568,7 @@ static void recv_done(struct ib_cq *cq, struct
ib_wc
*wc)} t->negotiation_requested = true; t->full_packet_received = true;+
enqueue_reassembly(t, recvmsg, 0);Is this a fix related to this patch?
Quoted 6 lines
Yes, this is required to receive the negotiation request from a client.
Before this patch, posting a buffer and waiting for the request is done in a function. Because
we have the reference of the buffer, we don't need to append it into the queue.
Okay.
Quoted 167 lines
Quoted 167 lines
Quoted 167 lines
wake_up_interruptible(&t->wait_status); break; case SMB_DIRECT_MSG_DATA_TRANSFER: {fs/ksmbd/transport_rdma.c +1
-7@@ -1594,19 +1595,13 @@ static int smb_direct_accept_client(struct smb_direct_transport *t) pr_err("error at rdma_accept: %d\n", ret);return ret; } - -wait_event_interruptible (t->wait_status, - t->status !=SMB_DIRECT_CS_NEW ); -if (t->status !=SMB_DIRECT_CS_CONNECTED ) -return -ENOTCONN ;return 0 ; } -static int smb_direct_negotiate (struct smb_direct_transport *t) +static int smb_direct_prepare_negotiation (struct smb_direct_transport *t) {int ret;struct smb_direct_recvmsg *recvmsg; -struct smb_direct_negotiate_req *req;recvmsg = get_free_recvmsg(t); if (!recvmsg)fs/ksmbd/transport_rdma.c +2
-6@@ -1616,44 +1611,20 @@ static int smb_direct_negotiate(struct smb_direct_transport *t) ret =smb_direct_post_recv (t, recvmsg);if (ret) {pr_err ("Can't post recv: %d\n" , ret); - goto out; + goto out_err; } t->negotiation_requested = false; ret =smb_direct_accept_client (t);if (ret) {pr_err ("Can't accept client\n" ); - goto out; + goto out_err; }smb_direct_post_recv_credits (&t->post_recv_credits_work.work); - -ksmbd_debug (RDMA ,"Waiting for SMB_DIRECT negotiate request\n" ); - ret =wait_event_interruptible_timeout (t->wait_status, - t->negotiation_requested|| - t->status ==
SMB_DIRECT_CS_DISCONNECTED,
-SMB_DIRECT_NEGOTIATE_TIMEOUT* HZ);fs/ksmbd/transport_rdma.c +10
-21-if (ret <=0 || t->status ==SMB_DIRECT_CS_DISCONNECTED ) { - ret = ret <0 ? ret : -ETIMEDOUT ; - goto out; - } - - ret =smb_direct_check_recvmsg (recvmsg); -if (ret == -ECONNABORTED ) - goto out; - - req = (struct smb_direct_negotiate_req *)recvmsg->packet; - t->max_recv_size =min_t (int, t->max_recv_size, -le32_to_cpu (req->preferred_send_size)); - t->max_send_size =min_t (int, t->max_send_size, -le32_to_cpu (req->max_receive_size)); - t->max_fragmented_send_size = -le32_to_cpu (req->max_fragmented_size); - - ret =smb_direct_send_negotiate_response (t, ret); -out: -if (recvmsg) -put_recvmsg (t, recvmsg); +return 0 ; +out_err: +put_recvmsg (t, recvmsg);return ret; }@@ -1890,6 +1861,47 @@ static int smb_direct_create_qpair(structsmb_direct_transport *t, static int smb_direct_prepare(struct ksmbd_transport *t) { struct smb_direct_transport *st = smb_trans_direct_transfort(t); +struct smb_direct_recvmsg *recvmsg; +struct smb_direct_negotiate_req *req; +int ret; + +ksmbd_debug (RDMA ,"Waiting for SMB_DIRECT negotiate request\n" ); + ret =wait_event_interruptible_timeout (st->wait_status, + st->negotiation_requested|| + st->status ==
SMB_DIRECT_CS_DISCONNECTED,
+SMB_DIRECT_NEGOTIATE_TIMEOUT* HZ);fs/ksmbd/transport_rdma.c +32
-0+if (ret <=0 || st->status ==SMB_DIRECT_CS_DISCONNECTED ) +return ret <0 ? ret : -ETIMEDOUT ; + + recvmsg =get_first_reassembly (st); +if (!recvmsg) +return -ECONNABORTED ; + + ret =smb_direct_check_recvmsg (recvmsg); +if (ret == -ECONNABORTED ) + goto out; + + req = (struct smb_direct_negotiate_req *)recvmsg->packet; + st->max_recv_size =min_t (int, st->max_recv_size, +le32_to_cpu (req->preferred_send_size)); + st->max_send_size =min_t (int, st->max_send_size, +le32_to_cpu (req->max_receive_size)); + st->max_fragmented_send_size = +le32_to_cpu (req->max_fragmented_size); + + ret =smb_direct_send_negotiate_response (st, ret); +out: +spin_lock_irq (&st->reassembly_queue_lock); + st->reassembly_queue_length--; +list_del (&recvmsg->list); +spin_unlock_irq (&st->reassembly_queue_lock); +put_recvmsg (st, recvmsg); + +return ret; +} + +static int smb_direct_connect (struct smb_direct_transport *st) +{int ret;struct ib_qp_cap qp_cap;@@ -1911,13 +1923,11 @@ static int smb_direct_prepare(struct
ksmbd_transport
*t)return ret; }fs/ksmbd/transport_rdma.c +2
-3- ret =smb_direct_negotiate (st); + ret =smb_direct_prepare_negotiation (st);if (ret) {pr_err ("Can't negotiate: %d\n" , ret);return ret; } - - st->status =SMB_DIRECT_CS_CONNECTED ;return 0 ; }@@ -1933,6 +1943,7 @@ static bool rdma_frwr_is_supported(struct ib_device_attr *attrs)static int smb_direct_handle_connect_request (struct rdma_cm_id *new_cm_id) {struct smb_direct_transport *t; +int ret;if (!rdma_frwr_is_supported(&new_cm_id->device->attrs)) { ksmbd_debug(RDMA,fs/ksmbd/transport_rdma.c +0
-0@@ -1945,11 +1956,17 @@ static intsmb_direct_handle_connect_request(struct
rdma_cm_id*new_cm_id)if (!t) return -ENOMEM;
Quoted 8 lines
Quoted 6 lines
fs/ksmbd/transport_rdma.c static int smb_direct_handle_connect_request(struct rdma_cm_id *new_cm_id) · line 1962
Quoted 4 lines
fs/ksmbd/transport_rdma.c +4
-0+ ret =smb_direct_connect (t); +if (ret) { +free_transport (t); +return ret;I think that it is better to use goto statement to out after freeing transport structure.
Okay, I will change it.
OK.
Thanks.
Quoted 24 lines
Quoted 11 lines
Quoted 11 lines
fs/ksmbd/transport_rdma.c +2
-0+ } + KSMBD_TRANS(t)->handler = kthread_run(ksmbd_conn_handler_loop, KSMBD_TRANS(t)->conn,"ksmbd:r%u",
smb_direct_port); if (IS_ERR(KSMBD_TRANS(t)->handler)) {fs/ksmbd/transport_rdma.c +1
-1-int ret =PTR_ERR (KSMBD_TRANS (t)->handler); + ret =PTR_ERR (KSMBD_TRANS (t)->handler);Quoted 6 lines
Quoted 6 lines
pr_err("Can't start thread\n"); free_transport(t);--
2.25.1--
Thanks,
Hyunchul