cifs: smbdirect: use the max_sge the hw reports
Subsystems (2)
Namjae Jeon
maintainerPATCH 1/1 cifs: smbdirect: use the max_sge the hw reports
In Soft-iWARP, smbdirect does not work in cifs client. The hardcoding max_sge is large in cifs, but need smaller value for soft-iWARP. Add SMBDIRECT_MIN_SGE macro as 6 and use the max_sge the hw reports instead of hardcoding 16 sge's.
--- fs/cifs/smbdirect.c | 15 ++++++++++----- fs/cifs/smbdirect.h | 3 ++- 2 files changed, 12 insertions(+), 6 deletions(-)
fs/cifs/smbdirect.c +10 -5
@@ -1518,7 +1518,7 @@ static int allocate_caches_and_workqueue(struct smbd_connection *info)static struct smbd_connection *_smbd_get_connection (struct TCP_Server_Info *server,struct sockaddr *dstaddr,int port) { -int rc; +int rc, max_sge;struct smbd_connection *info;struct rdma_conn_param conn_param;struct ib_qp_init_attr qp_attr;
@@ -1562,13 +1562,13 @@ static struct smbd_connection *_smbd_get_connection( info->max_receive_size = smbd_max_receive_size; info->keep_alive_interval = smbd_keep_alive_interval; -if (info->id->device->attrs.max_send_sge <SMBDIRECT_MAX_SGE ) { +if (info->id->device->attrs.max_send_sge <SMBDIRECT_MIN_SGE ) {log_rdma_event (ERR ,"warning: device max_send_sge = %d too small\n" , info->id->device->attrs.max_send_sge);log_rdma_event (ERR ,"Queue Pair creation may fail\n" ); } -if (info->id->device->attrs.max_recv_sge <SMBDIRECT_MAX_SGE ) { +if (info->id->device->attrs.max_recv_sge <SMBDIRECT_MIN_SGE ) {log_rdma_event (ERR ,"warning: device max_recv_sge = %d too small\n" , info->id->device->attrs.max_recv_sge);
@@ -1593,13 +1593,18 @@ static struct smbd_connection *_smbd_get_connection( goto alloc_cq_failed; } + max_sge =min3 (info->id->device->attrs.max_send_sge, + info->id->device->attrs.max_recv_sge, +SMBDIRECT_MAX_SGE ); + max_sge =max (max_sge,SMBDIRECT_MIN_SGE ); +memset (&qp_attr,0 ,sizeof (qp_attr)); qp_attr.event_handler = smbd_qp_async_error_upcall; qp_attr.qp_context = info; qp_attr.cap.max_send_wr = info->send_credit_target; qp_attr.cap.max_recv_wr = info->receive_credit_max; - qp_attr.cap.max_send_sge =SMBDIRECT_MAX_SGE ; - qp_attr.cap.max_recv_sge =SMBDIRECT_MAX_SGE ; + qp_attr.cap.max_send_sge = max_sge; + qp_attr.cap.max_recv_sge = max_sge; qp_attr.cap.max_inline_data =0 ; qp_attr.sq_sig_type =IB_SIGNAL_REQ_WR ; qp_attr.qp_type =IB_QPT_RC ;
fs/cifs/smbdirect.h +2 -1
@@ -225,7 +225,8 @@ struct smbd_buffer_descriptor_v1 {__le32 length; } __packed; -/* Default maximum number of SGEs in a RDMA send/recv */ +/* Default maximum/minimum number of SGEs in a RDMA send/recv */ +#define SMBDIRECT_MIN_SGE 6#define SMBDIRECT_MAX_SGE 16/* The context for a SMBD request */ struct smbd_request {
Signature · 1 line
2.25.1
Tom Talpey
reviewerQuoted 4 lines · Namjae Jeon
In Soft-iWARP, smbdirect does not work in cifs client. The hardcoding
max_sgeis large in cifs, but need smaller value for soft-iWARP. AddSMBDIRECT_MIN_SGEmacro as 6 and use themax_sgethe hw reports instead of hardcoding 16 sge's.
There is no issue in SoftiWARP, the bug is in ksmbd, so I think the message is incorrect. May I suggest:
"Use a more appropriate max_sge, and ensure it does not exceed the
RDMA provider's maximum. This enables ksmbd to function on SoftiWARP, among potentially others."
More comments below.
Quoted 40 lines
Cc Tom Talpey <[email protected]>Cc David Howells <[email protected]>Cc Hyunchul Lee <[email protected]>Cc Long Li <[email protected]>Signed-off-by Namjae Jeon <[email protected]>--- fs/cifs/smbdirect.c | 15 ++++++++++----- fs/cifs/smbdirect.h | 3 ++- 2 files changed, 12 insertions(+), 6 deletions(-)fs/cifs/smbdirect.c +3
-3@@ -1518,7 +1518,7 @@ static int allocate_caches_and_workqueue(struct smbd_connection *info)static struct smbd_connection *_smbd_get_connection (struct TCP_Server_Info *server,struct sockaddr *dstaddr,int port) { -int rc; +int rc, max_sge;struct smbd_connection *info;struct rdma_conn_param conn_param;struct ib_qp_init_attr qp_attr;@@ -1562,13 +1562,13 @@ static struct smbd_connection *_smbd_get_connection( info->max_receive_size = smbd_max_receive_size; info->keep_alive_interval = smbd_keep_alive_interval; -if (info->id->device->attrs.max_send_sge <SMBDIRECT_MAX_SGE ) { +if (info->id->device->attrs.max_send_sge <SMBDIRECT_MIN_SGE ) {log_rdma_event (ERR ,"warning: device max_send_sge = %d too small\n" , info->id->device->attrs.max_send_sge);log_rdma_event (ERR ,"Queue Pair creation may fail\n" ); } -if (info->id->device->attrs.max_recv_sge <SMBDIRECT_MAX_SGE ) { +if (info->id->device->attrs.max_recv_sge <SMBDIRECT_MIN_SGE ) {log_rdma_event (ERR ,"warning: device max_recv_sge = %d too small\n" , info->id->device->attrs.max_recv_sge);
fs/cifs/smbdirect.c static struct smbd_connection *_smbd_get_connection( · line 1594
Quoted 2 lines
fs/cifs/smbdirect.c +0
-0@@ -1593,13 +1593,18 @@ static struct smbd_connection *_smbd_get_connection( goto alloc_cq_failed; }
Why are the two conditions treated differently? It prints a rather vague warning if the send sge is exceeded, but it fails if the receive sge is too large.
I suggest failing fast in both cases, but the code gives no way for the user to correct the situation, SMBDIRECT_MIN_SGE is a hardcoded constant. That's the bug
IIRC, the ksmbd code requires 3 send sge's for send, and it needs 5 sge's when SMB3_TRANSFORM is needed. Why not provide a variable sge limit, depending on the session's requirement?
fs/cifs/smbdirect.c static struct smbd_connection *_smbd_get_connection( · line 1599
Quoted 4 lines
fs/cifs/smbdirect.c +4
-0+ max_sge =min3 (info->id->device->attrs.max_send_sge, + info->id->device->attrs.max_recv_sge, +SMBDIRECT_MAX_SGE ); + max_sge =max (max_sge,SMBDIRECT_MIN_SGE );
This is inaccurate. ksmbd's send sge requirement is not necessarily the same as its receive sge, likewise the RDMA provider's limit. There is no reason to limit one by the other, and they should be calculated independently.
What is the ksmbd receive sge requirement? Is it variable, like the send, depending on what protocol features are needed?
fs/cifs/smbdirect.c static struct smbd_connection *_smbd_get_connection( · line 1607
Quoted 10 lines
fs/cifs/smbdirect.c +3
-2+memset (&qp_attr,0 ,sizeof (qp_attr)); qp_attr.event_handler = smbd_qp_async_error_upcall; qp_attr.qp_context = info; qp_attr.cap.max_send_wr = info->send_credit_target; qp_attr.cap.max_recv_wr = info->receive_credit_max; - qp_attr.cap.max_send_sge =SMBDIRECT_MAX_SGE ; - qp_attr.cap.max_recv_sge =SMBDIRECT_MAX_SGE ; + qp_attr.cap.max_send_sge = max_sge; + qp_attr.cap.max_recv_sge = max_sge;
See previous comment.
Quoted 8 lines
fs/cifs/smbdirect.c +0
-0qp_attr.cap.max_inline_data =0 ; qp_attr.sq_sig_type =IB_SIGNAL_REQ_WR ; qp_attr.qp_type =IB_QPT_RC ;fs/cifs/smbdirect.h +0
-0
fs/cifs/smbdirect.h struct smbd_buffer_descriptor_v1 { · line 229
Quoted 6 lines
fs/cifs/smbdirect.h +2
-1@@ -225,7 +225,8 @@ struct smbd_buffer_descriptor_v1 {__le32 length; } __packed; -/* Default maximum number of SGEs in a RDMA send/recv */ +/* Default maximum/minimum number of SGEs in a RDMA send/recv */ +#define SMBDIRECT_MIN_SGE 6
See previous comment, and also, please justify the "6".
David Howells commented it appears to be "5", at least for send. I think with a small refactoring to allocate a more flexible header buffer, it could be even smaller.
I would hope the value for receive is "2", or less. But I haven't looked very deeply yet.
With sge's and an RDMA provider, the smaller the better. The adapter will always be more efficient in processing work requests. So doing this right is beneficial in many ways.
fs/cifs/smbdirect.h struct smbd_buffer_descriptor_v1 { · line 230
Quoted 1 line
fs/cifs/smbdirect.h +0
-0#define SMBDIRECT_MAX_SGE 16
While we're at it, please justify "16". Will ksmbd ever need so many?
fs/cifs/smbdirect.h struct smbd_buffer_descriptor_v1 { · line 232
Quoted 2 lines
fs/cifs/smbdirect.h +0
-0/* The context for a SMBD request */ struct smbd_request {
Tom.
Tom Talpey
reviewerOops, I typed "ksmbd" below, I meant "smbdirect client". However, fewer sge's are always better, and ksmbd may well have different requirements than "cifs", making a hardcoded value even less appropriate.
Quoted 134 lines · Tom Talpey
Quoted 4 lines · Namjae Jeon
In Soft-iWARP, smbdirect does not work in cifs client. The hardcoding
max_sgeis large in cifs, but need smaller value for soft-iWARP. AddSMBDIRECT_MIN_SGEmacro as 6 and use themax_sgethe hw reports instead of hardcoding 16 sge's.There is no issue in SoftiWARP, the bug is in ksmbd, so I think the message is incorrect. May I suggest:
"Use a more appropriate
max_sge, and ensure it does not exceed the RDMA provider's maximum. This enables ksmbd to function on SoftiWARP, among potentially others."More comments below.
Quoted 41 lines
Cc Tom Talpey <[email protected]>Cc David Howells <[email protected]>Cc Hyunchul Lee <[email protected]>Cc Long Li <[email protected]>Signed-off-by Namjae Jeon <[email protected]>---
fs/cifs/smbdirect.c| 15 ++++++++++-----
fs/cifs/smbdirect.h| 3 ++-2 files changed, 12 insertions(+), 6 deletions(-)fs/cifs/smbdirect.c +0
-0@@ -1518,7 +1518,7 @@ static int allocate_caches_and_workqueue(struct smbd_connection *info)static struct
smbd_connection*_smbd_get_connection(
structTCP_Server_Info*server, struct sockaddr *dstaddr, int port) {
- int rc;
+ int rc,max_sge;
structsmbd_connection*info;
structrdma_conn_paramconn_param;
structib_qp_init_attrqp_attr;fs/cifs/smbdirect.c +2
-2@@ -1562,13 +1562,13 @@ static struct smbd_connection *_smbd_get_connection ( info->max_receive_size = smbd_max_receive_size; info->keep_alive_interval = smbd_keep_alive_interval; - if (info->id->device->attrs.max_send_sge < SMBDIRECT_MAX_SGE) { + if (info->id->device->attrs.max_send_sge < SMBDIRECT_MIN_SGE) { log_rdma_event(ERR, "warning: device max_send_sge = %d too small\n", info->id->device->attrs.max_send_sge); log_rdma_event(ERR, "Queue Pair creation may fail\n"); } - if (info->id->device->attrs.max_recv_sge < SMBDIRECT_MAX_SGE) { + if (info->id->device->attrs.max_recv_sge < SMBDIRECT_MIN_SGE) { log_rdma_event(ERR, "warning: device max_recv_sge = %d too small\n", info->id->device->attrs.max_recv_sge);fs/cifs/smbdirect.c static struct smbd_connection · line 1595
Quoted 3 lines
fs/cifs/smbdirect.c +0
-0@@ -1593,13 +1593,18 @@ static struct smbd_connection *_smbd_get_connection( goto alloc_cq_failed; }Why are the two conditions treated differently? It prints a rather vague warning if the send sge is exceeded, but it fails if the receive sge is too large.
I suggest failing fast in both cases, but the code gives no way for the user to correct the situation,
SMBDIRECT_MIN_SGEis a hardcoded constant. That's the bugIIRC, the ksmbd code requires 3 send sge's for send, and it needs 5 sge's when
SMB3_TRANSFORMis needed. Why not provide a variable sge limit, depending on the session's requirement?fs/cifs/smbdirect.c static struct smbd_connection *_smbd_get_connection( · line 1599
Quoted 4 lines
fs/cifs/smbdirect.c +4
-0+ max_sge = min3(info->id->device->attrs.max_send_sge, + info->id->device->attrs.max_recv_sge, + SMBDIRECT_MAX_SGE); + max_sge = max(max_sge, SMBDIRECT_MIN_SGE);This is inaccurate. ksmbd's send sge requirement is not necessarily the same as its receive sge, likewise the RDMA provider's limit. There is no reason to limit one by the other, and they should be calculated independently.
What is the ksmbd receive sge requirement? Is it variable, like the send, depending on what protocol features are needed?
fs/cifs/smbdirect.c static struct smbd_connection *_smbd_get_connection( · line 1607
Quoted 10 lines
fs/cifs/smbdirect.c +3
-2+ memset(&qp_attr, 0, sizeof(qp_attr)); qp_attr.event_handler = smbd_qp_async_error_upcall; qp_attr.qp_context = info; qp_attr.cap.max_send_wr = info->send_credit_target; qp_attr.cap.max_recv_wr = info->receive_credit_max; - qp_attr.cap.max_send_sge = SMBDIRECT_MAX_SGE; - qp_attr.cap.max_recv_sge = SMBDIRECT_MAX_SGE; + qp_attr.cap.max_send_sge = max_sge; + qp_attr.cap.max_recv_sge = max_sge;See previous comment.
fs/cifs/smbdirect.h struct smbd_buffer_descriptor_v1 { · line 228
Quoted 13 lines
fs/cifs/smbdirect.c +0
-0qp_attr.cap.max_inline_data = 0; qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR; qp_attr.qp_type = IB_QPT_RC;fs/cifs/smbdirect.h +2
-1@@ -225,7 +225,8 @@ struct smbd_buffer_descriptor_v1 { __le32 length; } __packed; -/* Default maximum number of SGEs in a RDMA send/recv */ +/* Default maximum/minimum number of SGEs in a RDMA send/recv */ +#define SMBDIRECT_MIN_SGE 6See previous comment, and also, please justify the "6".
David Howells commented it appears to be "5", at least for send. I think with a small refactoring to allocate a more flexible header buffer, it could be even smaller.
I would hope the value for receive is "2", or less. But I haven't looked very deeply yet.
With sge's and an RDMA provider, the smaller the better. The adapter will always be more efficient in processing work requests. So doing this right is beneficial in many ways.
fs/cifs/smbdirect.h struct smbd_buffer_descriptor_v1 { · line 230
Quoted 1 line
fs/cifs/smbdirect.h +0
-0#define SMBDIRECT_MAX_SGE 16While we're at it, please justify "16". Will ksmbd ever need so many?
fs/cifs/smbdirect.h struct smbd_buffer_descriptor_v1 { · line 232
Quoted 2 lines
fs/cifs/smbdirect.h +0
-0/* The context for a SMBD request */ struct smbd_request {Tom.
Namjae Jeon
maintainer2022-08-03 23:34 GMT+09:00, Tom Talpey <[email protected]>:
Hi Tom,
Quoted 4 lines
Oops, I typed "ksmbd" below, I meant "smbdirect client". However, fewer sge's are always better, and ksmbd may well have different requirements than "cifs", making a hardcoded value even less appropriate.
Yes. Agreed. but I don't know if I can properly answer your questions. I have not looked deeply into the cifs smbdirect code, so I requested Long who is an author of smbdirect in cifs for a proper fix before. And it's still there. I just wrote a stopgap fix patch as on David's request. I expect Long or someone else with a deep look into cifs and smbdirect will give the right solution later.
Thanks!
Quoted 138 lines · Tom Talpey
Quoted 135 lines · Namjae Jeon
Quoted 4 lines
In Soft-iWARP, smbdirect does not work in cifs client. The hardcoding
max_sgeis large in cifs, but need smaller value for soft-iWARP. AddSMBDIRECT_MIN_SGEmacro as 6 and use themax_sgethe hw reports instead of hardcoding 16 sge's.There is no issue in SoftiWARP, the bug is in ksmbd, so I think the message is incorrect. May I suggest:
"Use a more appropriate
max_sge, and ensure it does not exceed the
RDMA provider's maximum. This enables ksmbd to function on SoftiWARP, among potentially others."More comments below.
Quoted 42 lines
Cc Tom Talpey <[email protected]>Cc David Howells <[email protected]>Cc Hyunchul Lee <[email protected]>Cc Long Li <[email protected]>Signed-off-by Namjae Jeon <[email protected]>--- fs/cifs/smbdirect.c | 15 ++++++++++----- fs/cifs/smbdirect.h | 3 ++- 2 files changed, 12 insertions(+), 6 deletions(-)fs/cifs/smbdirect.c +1
-1@@ -1518,7 +1518,7 @@ static int allocate_caches_and_workqueue(struct smbd_connection *info)static struct smbd_connection *_smbd_get_connection (struct TCP_Server_Info *server,struct sockaddr *dstaddr,int port) { -int rc; +int rc, max_sge;struct smbd_connection *info;struct rdma_conn_param conn_param; struct ib_qp_init_attr qp_attr;fs/cifs/smbdirect.c +2
-2@@ -1562,13 +1562,13 @@ static struct smbd_connection *_smbd_get_connection ( info->max_receive_size = smbd_max_receive_size; info->keep_alive_interval = smbd_keep_alive_interval; -if (info->id->device->attrs.max_send_sge <SMBDIRECT_MAX_SGE ) { +if (info->id->device->attrs.max_send_sge <SMBDIRECT_MIN_SGE ) {log_rdma_event (ERR ,"warning: device max_send_sge = %d too small\n" , info->id->device->attrs.max_send_sge);log_rdma_event (ERR ,"Queue Pair creation may fail\n" ); } -if (info->id->device->attrs.max_recv_sge <SMBDIRECT_MAX_SGE ) { +if (info->id->device->attrs.max_recv_sge <SMBDIRECT_MIN_SGE ) {log_rdma_event (ERR ,"warning: device max_recv_sge = %d too small\n" , info->id->device->attrs.max_recv_sge);fs/cifs/smbdirect.c static struct smbd_connection · line 1595
Quoted 3 lines
fs/cifs/smbdirect.c +0
-0@@ -1593,13 +1593,18 @@ static struct smbd_connection *_smbd_get_connection( goto alloc_cq_failed; }Why are the two conditions treated differently? It prints a rather vague warning if the send sge is exceeded, but it fails if the receive sge is too large.
I suggest failing fast in both cases, but the code gives no way for the user to correct the situation,
SMBDIRECT_MIN_SGEis a hardcoded constant. That's the bugIIRC, the ksmbd code requires 3 send sge's for send, and it needs 5 sge's when
SMB3_TRANSFORMis needed. Why not provide a variable sge limit, depending on the session's requirement?fs/cifs/smbdirect.c static struct smbd_connection *_smbd_get_connection( · line 1599
Quoted 4 lines
fs/cifs/smbdirect.c +4
-0+ max_sge =min3 (info->id->device->attrs.max_send_sge, + info->id->device->attrs.max_recv_sge, +SMBDIRECT_MAX_SGE ); + max_sge =max (max_sge,SMBDIRECT_MIN_SGE );This is inaccurate. ksmbd's send sge requirement is not necessarily the same as its receive sge, likewise the RDMA provider's limit. There is no reason to limit one by the other, and they should be calculated independently.
What is the ksmbd receive sge requirement? Is it variable, like the send, depending on what protocol features are needed?
fs/cifs/smbdirect.c static struct smbd_connection *_smbd_get_connection( · line 1607
Quoted 10 lines
fs/cifs/smbdirect.c +3
-2+memset (&qp_attr,0 ,sizeof (qp_attr)); qp_attr.event_handler = smbd_qp_async_error_upcall; qp_attr.qp_context = info; qp_attr.cap.max_send_wr = info->send_credit_target; qp_attr.cap.max_recv_wr = info->receive_credit_max; - qp_attr.cap.max_send_sge =SMBDIRECT_MAX_SGE ; - qp_attr.cap.max_recv_sge =SMBDIRECT_MAX_SGE ; + qp_attr.cap.max_send_sge = max_sge; + qp_attr.cap.max_recv_sge = max_sge;See previous comment.
fs/cifs/smbdirect.h struct smbd_buffer_descriptor_v1 { · line 228
Quoted 13 lines
fs/cifs/smbdirect.c +0
-0qp_attr.cap.max_inline_data =0 ; qp_attr.sq_sig_type =IB_SIGNAL_REQ_WR ; qp_attr.qp_type =IB_QPT_RC ;fs/cifs/smbdirect.h +2
-1@@ -225,7 +225,8 @@ struct smbd_buffer_descriptor_v1 {__le32 length; } __packed; -/* Default maximum number of SGEs in a RDMA send/recv */ +/* Default maximum/minimum number of SGEs in a RDMA send/recv */ +#define SMBDIRECT_MIN_SGE 6See previous comment, and also, please justify the "6".
David Howells commented it appears to be "5", at least for send. I think with a small refactoring to allocate a more flexible header buffer, it could be even smaller.
I would hope the value for receive is "2", or less. But I haven't looked very deeply yet.
With sge's and an RDMA provider, the smaller the better. The adapter will always be more efficient in processing work requests. So doing this right is beneficial in many ways.
fs/cifs/smbdirect.h struct smbd_buffer_descriptor_v1 { · line 230
Quoted 1 line
fs/cifs/smbdirect.h +0
-0#define SMBDIRECT_MAX_SGE 16While we're at it, please justify "16". Will ksmbd ever need so many?
fs/cifs/smbdirect.h struct smbd_buffer_descriptor_v1 { · line 232
Quoted 2 lines
fs/cifs/smbdirect.h +0
-0/* The context for a SMBD request */ struct smbd_request {Tom.
Tom Talpey
reviewerQuoted 6 lines · Namjae Jeon
2022-08-03 23:34 GMT+09:00, Tom Talpey <[email protected]>:
Hi Tom,Quoted 4 lines
Oops, I typed "ksmbd" below, I meant "smbdirect client". However, fewer sge's are always better, and ksmbd may well have different requirements than "cifs", making a hardcoded value even less appropriate.
Quoted 6 lines
Yes. Agreed. but I don't know if I can properly answer your questions. I have not looked deeply into the cifs smbdirect code, so I requested Long who is an author of smbdirect in cifs for a proper fix before. And it's still there. I just wrote a stopgap fix patch as on David's request. I expect Long or someone else with a deep look into cifs and smbdirect will give the right solution later.
Ok, well, I'll do it. The patch is a NAK from me, because it adds new, and incorrect, logic. If enabling SoftiWARP is the main goal, then simply one line would fix this:
#define SMBDIRECT_MAX_SGE 5
Quoted 140 lines
Thanks!
Quoted 138 lines · Tom Talpey
Quoted 135 lines · Namjae Jeon
Quoted 4 lines
In Soft-iWARP, smbdirect does not work in cifs client. The hardcoding
max_sgeis large in cifs, but need smaller value for soft-iWARP. AddSMBDIRECT_MIN_SGEmacro as 6 and use themax_sgethe hw reports instead of hardcoding 16 sge's.There is no issue in SoftiWARP, the bug is in ksmbd, so I think the message is incorrect. May I suggest:
"Use a more appropriate
max_sge, and ensure it does not exceed the
RDMA provider's maximum. This enables ksmbd to function on SoftiWARP, among potentially others."More comments below.
Quoted 42 lines
Cc Tom Talpey <[email protected]>Cc David Howells <[email protected]>Cc Hyunchul Lee <[email protected]>Cc Long Li <[email protected]>Signed-off-by Namjae Jeon <[email protected]>--- fs/cifs/smbdirect.c | 15 ++++++++++----- fs/cifs/smbdirect.h | 3 ++- 2 files changed, 12 insertions(+), 6 deletions(-)fs/cifs/smbdirect.c +1
-1@@ -1518,7 +1518,7 @@ static int allocate_caches_and_workqueue(struct smbd_connection *info)static struct smbd_connection *_smbd_get_connection (struct TCP_Server_Info *server,struct sockaddr *dstaddr,int port) { -int rc; +int rc, max_sge;struct smbd_connection *info;struct rdma_conn_param conn_param; struct ib_qp_init_attr qp_attr;fs/cifs/smbdirect.c +2
-2@@ -1562,13 +1562,13 @@ static struct smbd_connection *_smbd_get_connection ( info->max_receive_size = smbd_max_receive_size; info->keep_alive_interval = smbd_keep_alive_interval; -if (info->id->device->attrs.max_send_sge <SMBDIRECT_MAX_SGE ) { +if (info->id->device->attrs.max_send_sge <SMBDIRECT_MIN_SGE ) {log_rdma_event (ERR ,"warning: device max_send_sge = %d too small\n" , info->id->device->attrs.max_send_sge);log_rdma_event (ERR ,"Queue Pair creation may fail\n" ); } -if (info->id->device->attrs.max_recv_sge <SMBDIRECT_MAX_SGE ) { +if (info->id->device->attrs.max_recv_sge <SMBDIRECT_MIN_SGE ) {log_rdma_event (ERR ,"warning: device max_recv_sge = %d too small\n" , info->id->device->attrs.max_recv_sge);fs/cifs/smbdirect.c static struct smbd_connection · line 1595
Quoted 3 lines
fs/cifs/smbdirect.c +0
-0@@ -1593,13 +1593,18 @@ static struct smbd_connection *_smbd_get_connection( goto alloc_cq_failed; }Why are the two conditions treated differently? It prints a rather vague warning if the send sge is exceeded, but it fails if the receive sge is too large.
I suggest failing fast in both cases, but the code gives no way for the user to correct the situation,
SMBDIRECT_MIN_SGEis a hardcoded constant. That's the bugIIRC, the ksmbd code requires 3 send sge's for send, and it needs 5 sge's when
SMB3_TRANSFORMis needed. Why not provide a variable sge limit, depending on the session's requirement?fs/cifs/smbdirect.c static struct smbd_connection *_smbd_get_connection( · line 1599
Quoted 4 lines
fs/cifs/smbdirect.c +4
-0+ max_sge =min3 (info->id->device->attrs.max_send_sge, + info->id->device->attrs.max_recv_sge, +SMBDIRECT_MAX_SGE ); + max_sge =max (max_sge,SMBDIRECT_MIN_SGE );This is inaccurate. ksmbd's send sge requirement is not necessarily the same as its receive sge, likewise the RDMA provider's limit. There is no reason to limit one by the other, and they should be calculated independently.
What is the ksmbd receive sge requirement? Is it variable, like the send, depending on what protocol features are needed?
fs/cifs/smbdirect.c static struct smbd_connection *_smbd_get_connection( · line 1607
Quoted 10 lines
fs/cifs/smbdirect.c +3
-2+memset (&qp_attr,0 ,sizeof (qp_attr)); qp_attr.event_handler = smbd_qp_async_error_upcall; qp_attr.qp_context = info; qp_attr.cap.max_send_wr = info->send_credit_target; qp_attr.cap.max_recv_wr = info->receive_credit_max; - qp_attr.cap.max_send_sge =SMBDIRECT_MAX_SGE ; - qp_attr.cap.max_recv_sge =SMBDIRECT_MAX_SGE ; + qp_attr.cap.max_send_sge = max_sge; + qp_attr.cap.max_recv_sge = max_sge;See previous comment.
fs/cifs/smbdirect.h struct smbd_buffer_descriptor_v1 { · line 228
Quoted 13 lines
fs/cifs/smbdirect.c +0
-0qp_attr.cap.max_inline_data =0 ; qp_attr.sq_sig_type =IB_SIGNAL_REQ_WR ; qp_attr.qp_type =IB_QPT_RC ;fs/cifs/smbdirect.h +2
-1@@ -225,7 +225,8 @@ struct smbd_buffer_descriptor_v1 {__le32 length; } __packed; -/* Default maximum number of SGEs in a RDMA send/recv */ +/* Default maximum/minimum number of SGEs in a RDMA send/recv */ +#define SMBDIRECT_MIN_SGE 6See previous comment, and also, please justify the "6".
David Howells commented it appears to be "5", at least for send. I think with a small refactoring to allocate a more flexible header buffer, it could be even smaller.
I would hope the value for receive is "2", or less. But I haven't looked very deeply yet.
With sge's and an RDMA provider, the smaller the better. The adapter will always be more efficient in processing work requests. So doing this right is beneficial in many ways.
fs/cifs/smbdirect.h struct smbd_buffer_descriptor_v1 { · line 230
Quoted 1 line
fs/cifs/smbdirect.h +0
-0#define SMBDIRECT_MAX_SGE 16While we're at it, please justify "16". Will ksmbd ever need so many?
fs/cifs/smbdirect.h struct smbd_buffer_descriptor_v1 { · line 232
Quoted 2 lines
fs/cifs/smbdirect.h +0
-0/* The context for a SMBD request */ struct smbd_request {Tom.