RDMA/rxe: Bump up default maximum values used via uverbs
Subsystems (2)
v2 cover RDMA/rxe: Bump up default maximum values used via uverbs
Resubmitting my previous patch after kernel robot found that RXE_MAX_PKEYS is a u16 value and the value assigned would over flow and result in a zero value. I have reversed RXE_MAX_PKEYS to it original value.
Rao Shoaib (1):
RDMA/rxe: Bump up default maximum values used via uverbs
drivers/infiniband/sw/rxe/rxe_param.h | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-)
Signature · 1 line
2.27.0
PATCH v2 1/1 RDMA/rxe: Bump up default maximum values used via uverbs
From: Rao Shoaib <[email protected]>
In our internal testing we have found that the current maximum are too smalls. Ideally there should be no limits but currently maximum values are reported via ibv_query_device, so we have to keep maximum values but they have been made suffiently large.
Resubmitting after fixing an issue reported by test robot.
--- drivers/infiniband/sw/rxe/rxe_param.h | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-)
drivers/infiniband/sw/rxe/rxe_param.h +14 -12
@@ -9,6 +9,8 @@#include <uapi/rdma/rdma_user_rxe.h> +#define DEFAULT_MAX_VALUE (1 << 20) +static inline enum ib_mtu rxe_mtu_int_to_enum (int mtu) {if (mtu <256 )
@@ -37,7 +39,7 @@ static inline enum ib_mtu eth_mtu_int_to_enum(int mtu)enum rxe_device_param {RXE_MAX_MR_SIZE =-1ull ,RXE_PAGE_SIZE_CAP =0xfffff000 , -RXE_MAX_QP_WR =0x4000 , +RXE_MAX_QP_WR =DEFAULT_MAX_VALUE ,RXE_DEVICE_CAP_FLAGS =IB_DEVICE_BAD_PKEY_CNTR |IB_DEVICE_BAD_QKEY_CNTR |IB_DEVICE_AUTO_PATH_MIG
@@ -58,40 +60,40 @@ enum rxe_device_param {RXE_MAX_INLINE_DATA =RXE_MAX_WQE_SIZE -sizeof (struct rxe_send_wqe ),RXE_MAX_SGE_RD =32 , -RXE_MAX_CQ =16384 , +RXE_MAX_CQ =DEFAULT_MAX_VALUE ,RXE_MAX_LOG_CQE =15 , -RXE_MAX_PD =0x7ffc , +RXE_MAX_PD =DEFAULT_MAX_VALUE ,RXE_MAX_QP_RD_ATOM =128 ,RXE_MAX_RES_RD_ATOM =0x3f000 ,RXE_MAX_QP_INIT_RD_ATOM =128 ,RXE_MAX_MCAST_GRP =8192 ,RXE_MAX_MCAST_QP_ATTACH =56 ,RXE_MAX_TOT_MCAST_QP_ATTACH =0x70000 , -RXE_MAX_AH =100 , -RXE_MAX_SRQ_WR =0x4000 , +RXE_MAX_AH =DEFAULT_MAX_VALUE , +RXE_MAX_SRQ_WR =DEFAULT_MAX_VALUE ,RXE_MIN_SRQ_WR =1 ,RXE_MAX_SRQ_SGE =27 ,RXE_MIN_SRQ_SGE =1 ,RXE_MAX_FMR_PAGE_LIST_LEN =512 , -RXE_MAX_PKEYS =1 , +RXE_MAX_PKEYS =64 ,RXE_LOCAL_CA_ACK_DELAY =15 , -RXE_MAX_UCONTEXT =512 , +RXE_MAX_UCONTEXT =DEFAULT_MAX_VALUE ,RXE_NUM_PORT =1 , -RXE_MAX_QP =0x10000 , +RXE_MAX_QP =DEFAULT_MAX_VALUE ,RXE_MIN_QP_INDEX =16 , -RXE_MAX_QP_INDEX =0x00020000 , +RXE_MAX_QP_INDEX =0x00040000 , -RXE_MAX_SRQ =0x00001000 , +RXE_MAX_SRQ =DEFAULT_MAX_VALUE ,RXE_MIN_SRQ_INDEX =0x00020001 ,RXE_MAX_SRQ_INDEX =0x00040000 , -RXE_MAX_MR =0x00001000 , +RXE_MAX_MR =DEFAULT_MAX_VALUE ,RXE_MAX_MW =0x00001000 ,RXE_MIN_MR_INDEX =0x00000001 , -RXE_MAX_MR_INDEX =0x00010000 , +RXE_MAX_MR_INDEX =0x00040000 ,RXE_MIN_MW_INDEX =0x00010001 , RXE_MAX_MW_INDEX = 0x00020000,
Signature · 1 line
2.27.0
Zhu Yanjun
maintainerQuoted 23 lines · Rao Shoaib
From: Rao Shoaib <[email protected]>
In our internal testing we have found that the current maximum are too smalls. Ideally there should be no limits but currently maximum values are reported via
ibv_query_device, so we have to keep maximum values but they have been made suffiently large.Resubmitting after fixing an issue reported by test robot.
Reported-by kernel test robot <[email protected]>Signed-off-by Rao Shoaib <[email protected]>--- drivers/infiniband/sw/rxe/rxe_param.h | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-)drivers/infiniband/sw/rxe/rxe_param.h +0
-0
drivers/infiniband/sw/rxe/rxe_param.h line 12
Quoted 4 lines
drivers/infiniband/sw/rxe/rxe_param.h +1
-0@@ -9,6 +9,8 @@#include <uapi/rdma/rdma_user_rxe.h> +#define DEFAULT_MAX_VALUE (1 << 20)
Can you let me know the link in which the above value is discussed?
Thanks,
Zhu Yanjun
Quoted 68 lines
drivers/infiniband/sw/rxe/rxe_param.h +13
-13+static inline enum ib_mtu rxe_mtu_int_to_enum (int mtu) {if (mtu <256 )@@ -37,7 +39,7 @@ static inline enum ib_mtu eth_mtu_int_to_enum(int mtu)enum rxe_device_param {RXE_MAX_MR_SIZE =-1ull ,RXE_PAGE_SIZE_CAP =0xfffff000 , -RXE_MAX_QP_WR =0x4000 , +RXE_MAX_QP_WR =DEFAULT_MAX_VALUE ,RXE_DEVICE_CAP_FLAGS =IB_DEVICE_BAD_PKEY_CNTR |IB_DEVICE_BAD_QKEY_CNTR |IB_DEVICE_AUTO_PATH_MIG @@ -58,40 +60,40 @@ enum rxe_device_param {RXE_MAX_INLINE_DATA =RXE_MAX_WQE_SIZE -sizeof (struct rxe_send_wqe ),RXE_MAX_SGE_RD =32 , -RXE_MAX_CQ =16384 , +RXE_MAX_CQ =DEFAULT_MAX_VALUE ,RXE_MAX_LOG_CQE =15 , -RXE_MAX_PD =0x7ffc , +RXE_MAX_PD =DEFAULT_MAX_VALUE ,RXE_MAX_QP_RD_ATOM =128 ,RXE_MAX_RES_RD_ATOM =0x3f000 ,RXE_MAX_QP_INIT_RD_ATOM =128 ,RXE_MAX_MCAST_GRP =8192 ,RXE_MAX_MCAST_QP_ATTACH =56 ,RXE_MAX_TOT_MCAST_QP_ATTACH =0x70000 , -RXE_MAX_AH =100 , -RXE_MAX_SRQ_WR =0x4000 , +RXE_MAX_AH =DEFAULT_MAX_VALUE , +RXE_MAX_SRQ_WR =DEFAULT_MAX_VALUE ,RXE_MIN_SRQ_WR =1 ,RXE_MAX_SRQ_SGE =27 ,RXE_MIN_SRQ_SGE =1 ,RXE_MAX_FMR_PAGE_LIST_LEN =512 , -RXE_MAX_PKEYS =1 , +RXE_MAX_PKEYS =64 ,RXE_LOCAL_CA_ACK_DELAY =15 , -RXE_MAX_UCONTEXT =512 , +RXE_MAX_UCONTEXT =DEFAULT_MAX_VALUE ,RXE_NUM_PORT =1 , -RXE_MAX_QP =0x10000 , +RXE_MAX_QP =DEFAULT_MAX_VALUE ,RXE_MIN_QP_INDEX =16 , -RXE_MAX_QP_INDEX =0x00020000 , +RXE_MAX_QP_INDEX =0x00040000 , -RXE_MAX_SRQ =0x00001000 , +RXE_MAX_SRQ =DEFAULT_MAX_VALUE ,RXE_MIN_SRQ_INDEX =0x00020001 ,RXE_MAX_SRQ_INDEX =0x00040000 , -RXE_MAX_MR =0x00001000 , +RXE_MAX_MR =DEFAULT_MAX_VALUE ,RXE_MAX_MW =0x00001000 ,RXE_MIN_MR_INDEX =0x00000001 , -RXE_MAX_MR_INDEX =0x00010000 , +RXE_MAX_MR_INDEX =0x00040000 ,RXE_MIN_MW_INDEX =0x00010001 , RXE_MAX_MW_INDEX = 0x00020000, --2.27.0
Following is a link
https://marc.info/?l=linux-rdma&m=162395437604846&w=2
Or just search for my name in the archive.
Do you see any issues with this value?
Shoaib
Quoted 100 lines · Zhu Yanjun
Quoted 22 lines · Rao Shoaib
From: Rao Shoaib <[email protected]>
In our internal testing we have found that the current maximum are too smalls. Ideally there should be no limits but currently maximum values are reported via
ibv_query_device, so we have to keep maximum values but they have been made suffiently large.Resubmitting after fixing an issue reported by test robot.
Reported-by kernel test robot <[email protected]>Signed-off-by Rao Shoaib <[email protected]>--- drivers/infiniband/sw/rxe/rxe_param.h | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-)drivers/infiniband/sw/rxe/rxe_param.h +0
-0drivers/infiniband/sw/rxe/rxe_param.h line 12
Quoted 4 lines
drivers/infiniband/sw/rxe/rxe_param.h +1
-0@@ -9,6 +9,8 @@#include <uapi/rdma/rdma_user_rxe.h> +#define DEFAULT_MAX_VALUE (1 << 20)Can you let me know the link in which the above value is discussed?
Thanks,
Zhu YanjunQuoted 68 lines
drivers/infiniband/sw/rxe/rxe_param.h +13
-13+static inline enum ib_mtu rxe_mtu_int_to_enum (int mtu) {if (mtu <256 )@@ -37,7 +39,7 @@ static inline enum ib_mtu eth_mtu_int_to_enum(int mtu)enum rxe_device_param {RXE_MAX_MR_SIZE =-1ull ,RXE_PAGE_SIZE_CAP =0xfffff000 , -RXE_MAX_QP_WR =0x4000 , +RXE_MAX_QP_WR =DEFAULT_MAX_VALUE ,RXE_DEVICE_CAP_FLAGS =IB_DEVICE_BAD_PKEY_CNTR |IB_DEVICE_BAD_QKEY_CNTR |IB_DEVICE_AUTO_PATH_MIG @@ -58,40 +60,40 @@ enum rxe_device_param {RXE_MAX_INLINE_DATA =RXE_MAX_WQE_SIZE -sizeof (struct rxe_send_wqe ),RXE_MAX_SGE_RD =32 , -RXE_MAX_CQ =16384 , +RXE_MAX_CQ =DEFAULT_MAX_VALUE ,RXE_MAX_LOG_CQE =15 , -RXE_MAX_PD =0x7ffc , +RXE_MAX_PD =DEFAULT_MAX_VALUE ,RXE_MAX_QP_RD_ATOM =128 ,RXE_MAX_RES_RD_ATOM =0x3f000 ,RXE_MAX_QP_INIT_RD_ATOM =128 ,RXE_MAX_MCAST_GRP =8192 ,RXE_MAX_MCAST_QP_ATTACH =56 ,RXE_MAX_TOT_MCAST_QP_ATTACH =0x70000 , -RXE_MAX_AH =100 , -RXE_MAX_SRQ_WR =0x4000 , +RXE_MAX_AH =DEFAULT_MAX_VALUE , +RXE_MAX_SRQ_WR =DEFAULT_MAX_VALUE ,RXE_MIN_SRQ_WR =1 ,RXE_MAX_SRQ_SGE =27 ,RXE_MIN_SRQ_SGE =1 ,RXE_MAX_FMR_PAGE_LIST_LEN =512 , -RXE_MAX_PKEYS =1 , +RXE_MAX_PKEYS =64 ,RXE_LOCAL_CA_ACK_DELAY =15 , -RXE_MAX_UCONTEXT =512 , +RXE_MAX_UCONTEXT =DEFAULT_MAX_VALUE ,RXE_NUM_PORT =1 , -RXE_MAX_QP =0x10000 , +RXE_MAX_QP =DEFAULT_MAX_VALUE ,RXE_MIN_QP_INDEX =16 , -RXE_MAX_QP_INDEX =0x00020000 , +RXE_MAX_QP_INDEX =0x00040000 , -RXE_MAX_SRQ =0x00001000 , +RXE_MAX_SRQ =DEFAULT_MAX_VALUE ,RXE_MIN_SRQ_INDEX =0x00020001 ,RXE_MAX_SRQ_INDEX =0x00040000 , -RXE_MAX_MR =0x00001000 , +RXE_MAX_MR =DEFAULT_MAX_VALUE ,RXE_MAX_MW =0x00001000 ,RXE_MIN_MR_INDEX =0x00000001 , -RXE_MAX_MR_INDEX =0x00010000 , +RXE_MAX_MR_INDEX =0x00040000 ,RXE_MIN_MW_INDEX =0x00010001 , RXE_MAX_MW_INDEX = 0x00020000, --2.27.0
Zhu Yanjun
maintainerQuoted 8 lines · Shoaib Rao
Following is a link
https://marc.info/?l=linux-rdma&m=162395437604846&w=2
Or just search for my name in the archive.
Do you see any issues with this value?
After this commit is applied, I confronted the following problem "
[ 639.943561] rdma_rxe: unloaded [ 679.717143] rdma_rxe: loaded [ 691.721055] rdma_rxe: not enough indices for max_elem
"
Not sure if this problem is introduced by this commit. Please help to check this problem.
Thanks a lot.
Zhu Yanjun
Quoted 104 lines
Shoaib
Quoted 100 lines · Zhu Yanjun
Quoted 22 lines · Rao Shoaib
From: Rao Shoaib <[email protected]>
In our internal testing we have found that the current maximum are too smalls. Ideally there should be no limits but currently maximum values are reported via
ibv_query_device, so we have to keep maximum values but they have been made suffiently large.Resubmitting after fixing an issue reported by test robot.
Reported-by kernel test robot <[email protected]>Signed-off-by Rao Shoaib <[email protected]>--- drivers/infiniband/sw/rxe/rxe_param.h | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-)drivers/infiniband/sw/rxe/rxe_param.h +0
-0drivers/infiniband/sw/rxe/rxe_param.h line 12
Quoted 4 lines
drivers/infiniband/sw/rxe/rxe_param.h +1
-0@@ -9,6 +9,8 @@#include <uapi/rdma/rdma_user_rxe.h> +#define DEFAULT_MAX_VALUE (1 << 20)Can you let me know the link in which the above value is discussed?
Thanks,
Zhu YanjunQuoted 68 lines
drivers/infiniband/sw/rxe/rxe_param.h +13
-13+static inline enum ib_mtu rxe_mtu_int_to_enum (int mtu) {if (mtu <256 )@@ -37,7 +39,7 @@ static inline enum ib_mtu eth_mtu_int_to_enum(int mtu)enum rxe_device_param {RXE_MAX_MR_SIZE =-1ull ,RXE_PAGE_SIZE_CAP =0xfffff000 , -RXE_MAX_QP_WR =0x4000 , +RXE_MAX_QP_WR =DEFAULT_MAX_VALUE ,RXE_DEVICE_CAP_FLAGS =IB_DEVICE_BAD_PKEY_CNTR |IB_DEVICE_BAD_QKEY_CNTR |IB_DEVICE_AUTO_PATH_MIG @@ -58,40 +60,40 @@ enum rxe_device_param {RXE_MAX_INLINE_DATA =RXE_MAX_WQE_SIZE -sizeof (struct rxe_send_wqe ),RXE_MAX_SGE_RD =32 , -RXE_MAX_CQ =16384 , +RXE_MAX_CQ =DEFAULT_MAX_VALUE ,RXE_MAX_LOG_CQE =15 , -RXE_MAX_PD =0x7ffc , +RXE_MAX_PD =DEFAULT_MAX_VALUE ,RXE_MAX_QP_RD_ATOM =128 ,RXE_MAX_RES_RD_ATOM =0x3f000 ,RXE_MAX_QP_INIT_RD_ATOM =128 ,RXE_MAX_MCAST_GRP =8192 ,RXE_MAX_MCAST_QP_ATTACH =56 ,RXE_MAX_TOT_MCAST_QP_ATTACH =0x70000 , -RXE_MAX_AH =100 , -RXE_MAX_SRQ_WR =0x4000 , +RXE_MAX_AH =DEFAULT_MAX_VALUE , +RXE_MAX_SRQ_WR =DEFAULT_MAX_VALUE ,RXE_MIN_SRQ_WR =1 ,RXE_MAX_SRQ_SGE =27 ,RXE_MIN_SRQ_SGE =1 ,RXE_MAX_FMR_PAGE_LIST_LEN =512 , -RXE_MAX_PKEYS =1 , +RXE_MAX_PKEYS =64 ,RXE_LOCAL_CA_ACK_DELAY =15 , -RXE_MAX_UCONTEXT =512 , +RXE_MAX_UCONTEXT =DEFAULT_MAX_VALUE ,RXE_NUM_PORT =1 , -RXE_MAX_QP =0x10000 , +RXE_MAX_QP =DEFAULT_MAX_VALUE ,RXE_MIN_QP_INDEX =16 , -RXE_MAX_QP_INDEX =0x00020000 , +RXE_MAX_QP_INDEX =0x00040000 , -RXE_MAX_SRQ =0x00001000 , +RXE_MAX_SRQ =DEFAULT_MAX_VALUE ,RXE_MIN_SRQ_INDEX =0x00020001 ,RXE_MAX_SRQ_INDEX =0x00040000 , -RXE_MAX_MR =0x00001000 , +RXE_MAX_MR =DEFAULT_MAX_VALUE ,RXE_MAX_MW =0x00001000 ,RXE_MIN_MR_INDEX =0x00000001 , -RXE_MAX_MR_INDEX =0x00010000 , +RXE_MAX_MR_INDEX =0x00040000 ,RXE_MIN_MW_INDEX =0x00010001 , RXE_MAX_MW_INDEX = 0x00020000, --2.27.0
A message that is not in the archive (sent off-list or not yet received): [email protected]
Zhu Yanjun
maintainerQuoted 59 lines · Shoaib Rao
On 7/15/21 9:07 PM, Zhu Yanjun wrote:
On Fri, Jul 16, 2021 at 12:44 AM Shoaib Rao <[email protected]> wrote:
Following is a link
Or just search for my name in the archive.
Do you see any issues with this value?
After this commit is applied, I confronted the following problem "
[ 639.943561] rdma_rxe: unloaded [ 679.717143] rdma_rxe: loaded [ 691.721055] rdma_rxe: not enough indices for max_elem"
Not sure if this problem is introduced by this commit. Please help to check this problem.
Thanks a lot.Zhu Yanjun
I do not see the issue.
[root@ca-dev66 rxe]# lsmod | grep
rdma_rxerdma_rxe 126976 0 ip6_udp_tunnel 16384 1 rdma_rxe udp_tunnel 20480 1 rdma_rxe ib_uverbs 147456 3 rdma_rxe,rdma_ucm,mlx5_ib ib_core 364544 10 rdma_cm,ib_ipoib,rdma_rxe,rds_rdma,iw_cm,ib_umad,rdma_ucm,ib_uverbs,mlx5_ib,ib_cmI am using gdb to dump the values
(gdb) ptype
RXE_MAX_INLINE_DATA
type = enumrxe_device_param{RXE_MAX_MR_SIZE= -1,RXE_PAGE_SIZE_CAP = 4294963200, RXE_MAX_QP_WR = 1048576, RXE_DEVICE_CAP_FLAGS = 137466362998, RXE_MAX_SGE = 32, RXE_MAX_WQE_SIZE = 720, RXE_MAX_INLINE_DATA = 512, RXE_MAX_SGE_RD = 32, RXE_MAX_CQ = 1048576, RXE_MAX_LOG_CQE = 15, RXE_MAX_PD = 1048576, RXE_MAX_QP_RD_ATOM = 128, RXE_MAX_RES_RD_ATOM = 258048, RXE_MAX_QP_INIT_RD_ATOM = 128, RXE_MAX_MCAST_GRP = 8192, RXE_MAX_MCAST_QP_ATTACH = 56, RXE_MAX_TOT_MCAST_QP_ATTACH = 458752, RXE_MAX_AH = 1048576, RXE_MAX_SRQ_WR = 1048576, RXE_MIN_SRQ_WR = 1, RXE_MAX_SRQ_SGE = 27, RXE_MIN_SRQ_SGE = 1, RXE_MAX_FMR_PAGE_LIST_LEN = 512, RXE_MAX_PKEYS = 64, RXE_LOCAL_CA_ACK_DELAY = 15, RXE_MAX_UCONTEXT = 1048576, RXE_NUM_PORT = 1, RXE_MAX_QP = 1048576, RXE_MIN_QP_INDEX = 16, RXE_MAX_QP_INDEX = 262144, RXE_MAX_SRQ = 1048576, RXE_MIN_SRQ_INDEX = 131073, RXE_MAX_SRQ_INDEX = 262144, RXE_MAX_MR = 1048576, RXE_MAX_MW = 4096, RXE_MIN_MR_INDEX = 1, RXE_MAX_MR_INDEX = 262144, RXE_MIN_MW_INDEX = 65537, RXE_MAX_MW_INDEX = 131072, RXE_MAX_PKT_PER_ACK = 64, RXE_MAX_UNACKED_PSNS = 128, RXE_INFLIGHT_SKBS_PER_QP_HIGH = 64, RXE_INFLIGHT_SKBS_PER_QP_LOW = 16, RXE_NSEC_ARB_TIMER_DELAY = 200, RXE_VENDOR_ID = 16777215}
Quoted 1 line
It is possible that you are changing some values.
I git clone the source code from
https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git again. And the first 3 commits are as below. One is your commit. "6163ad3208c8 (HEAD -> for-next) RDMA/rxe: Bump up default maximum values used via uverbs8c1b4316c3fa (origin/for-next, origin/HEAD) RDMA/efa: Split hardware stats to device and port stats916071185b17 MAINTAINERS: Update maintainers of HiSilicon RoCE "
And when I tried to add a new rxe0,
"
error: Invalid argument
"
And the dmesg logs are as below:
# dmesg " [ 70.782302] e1000: enp0s8 NIC Link is Up 1000 Mbps Full Duplex, Flow Control: RX [ 70.782744] IPv6: ADDRCONF(NETDEV_CHANGE): enp0s8: link becomes ready [ 79.467652] rdma_rxe: loaded [ 79.468348] rdma_rxe: not enough indices for max_elem [ 79.468356] rdma_rxe: failed to add enp0s8 "
Please follow my steps to reproduce this problem again.
Zhu Yanjun
Quoted 109 lines
Shoaib
Shoaib
On 7/14/21 10:02 PM, Zhu Yanjun wrote:
On Wed, Jul 14, 2021 at 4:36 PM Rao Shoaib <[email protected]> wrote:
From: Rao Shoaib <[email protected]>
In our internal testing we have found that the current maximum are too smalls. Ideally there should be no limits but currently maximum values are reported via
ibv_query_device, so we have to keep maximum values but they have been made suffiently large.Resubmitting after fixing an issue reported by test robot.
Reported-by kernel test robot <[email protected]>Signed-off-by Rao Shoaib <[email protected]>--- drivers/infiniband/sw/rxe/rxe_param.h | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-)drivers/infiniband/sw/rxe/rxe_param.h +1
-0@@ -9,6 +9,8 @@#include <uapi/rdma/rdma_user_rxe.h> +#define DEFAULT_MAX_VALUE (1 << 20)Can you let me know the link in which the above value is discussed?
Thanks,
Zhu Yanjundrivers/infiniband/sw/rxe/rxe_param.h +13
-13+static inline enum ib_mtu rxe_mtu_int_to_enum (int mtu) {if (mtu <256 )@@ -37,7 +39,7 @@ static inline enum ib_mtu eth_mtu_int_to_enum(int mtu)enum rxe_device_param {RXE_MAX_MR_SIZE =-1ull ,RXE_PAGE_SIZE_CAP =0xfffff000 , -RXE_MAX_QP_WR =0x4000 , +RXE_MAX_QP_WR =DEFAULT_MAX_VALUE ,RXE_DEVICE_CAP_FLAGS =IB_DEVICE_BAD_PKEY_CNTR |IB_DEVICE_BAD_QKEY_CNTR |IB_DEVICE_AUTO_PATH_MIG @@ -58,40 +60,40 @@ enum rxe_device_param {RXE_MAX_INLINE_DATA =RXE_MAX_WQE_SIZE -sizeof (struct rxe_send_wqe ),RXE_MAX_SGE_RD =32 , -RXE_MAX_CQ =16384 , +RXE_MAX_CQ =DEFAULT_MAX_VALUE ,RXE_MAX_LOG_CQE =15 , -RXE_MAX_PD =0x7ffc , +RXE_MAX_PD =DEFAULT_MAX_VALUE ,RXE_MAX_QP_RD_ATOM =128 ,RXE_MAX_RES_RD_ATOM =0x3f000 ,RXE_MAX_QP_INIT_RD_ATOM =128 ,RXE_MAX_MCAST_GRP =8192 ,RXE_MAX_MCAST_QP_ATTACH =56 ,RXE_MAX_TOT_MCAST_QP_ATTACH =0x70000 , -RXE_MAX_AH =100 , -RXE_MAX_SRQ_WR =0x4000 , +RXE_MAX_AH =DEFAULT_MAX_VALUE , +RXE_MAX_SRQ_WR =DEFAULT_MAX_VALUE ,RXE_MIN_SRQ_WR =1 ,RXE_MAX_SRQ_SGE =27 ,RXE_MIN_SRQ_SGE =1 ,RXE_MAX_FMR_PAGE_LIST_LEN =512 , -RXE_MAX_PKEYS =1 , +RXE_MAX_PKEYS =64 ,RXE_LOCAL_CA_ACK_DELAY =15 , -RXE_MAX_UCONTEXT =512 , +RXE_MAX_UCONTEXT =DEFAULT_MAX_VALUE ,RXE_NUM_PORT =1 , -RXE_MAX_QP =0x10000 , +RXE_MAX_QP =DEFAULT_MAX_VALUE ,RXE_MIN_QP_INDEX =16 , -RXE_MAX_QP_INDEX =0x00020000 , +RXE_MAX_QP_INDEX =0x00040000 , -RXE_MAX_SRQ =0x00001000 , +RXE_MAX_SRQ =DEFAULT_MAX_VALUE ,RXE_MIN_SRQ_INDEX =0x00020001 ,RXE_MAX_SRQ_INDEX =0x00040000 , -RXE_MAX_MR =0x00001000 , +RXE_MAX_MR =DEFAULT_MAX_VALUE ,RXE_MAX_MW =0x00001000 ,RXE_MIN_MR_INDEX =0x00000001 , -RXE_MAX_MR_INDEX =0x00010000 , +RXE_MAX_MR_INDEX =0x00040000 ,RXE_MIN_MW_INDEX =0x00010001 , RXE_MAX_MW_INDEX = 0x00020000, --2.27.0
Your urls are garbled, so I can not make any sense out of them.
[root@ca-dev141 linux]# git remote --v
origin https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git (fetch) origin https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git (push)
[root@ca-dev141 linux]# git show 8c1b4316c3fa
fatal: ambiguous argument '8c1b4316c3fa': unknown revision or path not in the working tree.
[root@ca-dev141 linux]# git log --grep='RDMA/efa: Split hardware' [root@ca-dev141 linux]#
[root@ca-dev141 linux]# git describe
v5.14-rc1-17-gc06676b5953b
You need to do a little bit more work on your end and tell me which value rxe is complaining about. Also provide a kernel version that you are using as I have provided above.
Shoaib
Quoted 200 lines · Zhu Yanjun
Quoted 58 lines · Shoaib Rao
On 7/15/21 9:07 PM, Zhu Yanjun wrote:
On Fri, Jul 16, 2021 at 12:44 AM Shoaib Rao <[email protected]> wrote:
Following is a link
Or just search for my name in the archive.
Do you see any issues with this value?
After this commit is applied, I confronted the following problem "
[ 639.943561] rdma_rxe: unloaded [ 679.717143] rdma_rxe: loaded [ 691.721055] rdma_rxe: not enough indices for max_elem"
Not sure if this problem is introduced by this commit. Please help to check this problem.
Thanks a lot.Zhu Yanjun
I do not see the issue.
[root@ca-dev66 rxe]# lsmod | grep
rdma_rxerdma_rxe 126976 0 ip6_udp_tunnel 16384 1 rdma_rxe udp_tunnel 20480 1 rdma_rxe ib_uverbs 147456 3 rdma_rxe,rdma_ucm,mlx5_ib ib_core 364544 10 rdma_cm,ib_ipoib,rdma_rxe,rds_rdma,iw_cm,ib_umad,rdma_ucm,ib_uverbs,mlx5_ib,ib_cmI am using gdb to dump the values
(gdb) ptype
RXE_MAX_INLINE_DATA
type = enumrxe_device_param{RXE_MAX_MR_SIZE= -1,RXE_PAGE_SIZE_CAP = 4294963200, RXE_MAX_QP_WR = 1048576, RXE_DEVICE_CAP_FLAGS = 137466362998, RXE_MAX_SGE = 32, RXE_MAX_WQE_SIZE = 720, RXE_MAX_INLINE_DATA = 512, RXE_MAX_SGE_RD = 32, RXE_MAX_CQ = 1048576, RXE_MAX_LOG_CQE = 15, RXE_MAX_PD = 1048576, RXE_MAX_QP_RD_ATOM = 128, RXE_MAX_RES_RD_ATOM = 258048, RXE_MAX_QP_INIT_RD_ATOM = 128, RXE_MAX_MCAST_GRP = 8192, RXE_MAX_MCAST_QP_ATTACH = 56, RXE_MAX_TOT_MCAST_QP_ATTACH = 458752, RXE_MAX_AH = 1048576, RXE_MAX_SRQ_WR = 1048576, RXE_MIN_SRQ_WR = 1, RXE_MAX_SRQ_SGE = 27, RXE_MIN_SRQ_SGE = 1, RXE_MAX_FMR_PAGE_LIST_LEN = 512, RXE_MAX_PKEYS = 64, RXE_LOCAL_CA_ACK_DELAY = 15, RXE_MAX_UCONTEXT = 1048576, RXE_NUM_PORT = 1, RXE_MAX_QP = 1048576, RXE_MIN_QP_INDEX = 16, RXE_MAX_QP_INDEX = 262144, RXE_MAX_SRQ = 1048576, RXE_MIN_SRQ_INDEX = 131073, RXE_MAX_SRQ_INDEX = 262144, RXE_MAX_MR = 1048576, RXE_MAX_MW = 4096, RXE_MIN_MR_INDEX = 1, RXE_MAX_MR_INDEX = 262144, RXE_MIN_MW_INDEX = 65537, RXE_MAX_MW_INDEX = 131072, RXE_MAX_PKT_PER_ACK = 64, RXE_MAX_UNACKED_PSNS = 128, RXE_INFLIGHT_SKBS_PER_QP_HIGH = 64, RXE_INFLIGHT_SKBS_PER_QP_LOW = 16, RXE_NSEC_ARB_TIMER_DELAY = 200, RXE_VENDOR_ID = 16777215}Quoted 1 line
It is possible that you are changing some values.
I git clone the source code from
https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git__;!!ACWV5N9M2RV99hQ!efPsL9fikzm6Ob1-zsmbsAzPkfqRpxYkp9oJXF9csPOJgoei9njj8RmA5IyuH447$ again. And the first 3 commits are as below. One is your commit. " 6163ad3208c8 (HEAD -> for-next) RDMA/rxe: Bump up default maximum values used via uverbs 8c1b4316c3fa (origin/for-next, origin/HEAD) RDMA/efa: Split hardware stats to device and port stats 916071185b17 MAINTAINERS: Update maintainers of HiSilicon RoCE "And when I tried to add a new rxe0,
"
error: Invalid argument
"
And the dmesg logs are as below:# dmesg " [ 70.782302] e1000: enp0s8 NIC Link is Up 1000 Mbps Full Duplex, Flow Control: RX [ 70.782744] IPv6: ADDRCONF(NETDEV_CHANGE): enp0s8: link becomes ready [ 79.467652] rdma_rxe: loaded [ 79.468348] rdma_rxe: not enough indices for max_elem [ 79.468356] rdma_rxe: failed to add enp0s8 "Please follow my steps to reproduce this problem again.
Zhu Yanjun
Quoted 108 lines
Shoaib
Shoaib
On 7/14/21 10:02 PM, Zhu Yanjun wrote:
On Wed, Jul 14, 2021 at 4:36 PM Rao Shoaib <[email protected]> wrote:
From: Rao Shoaib <[email protected]>
In our internal testing we have found that the current maximum are too smalls. Ideally there should be no limits but currently maximum values are reported via
ibv_query_device, so we have to keep maximum values but they have been made suffiently large.Resubmitting after fixing an issue reported by test robot.
Reported-by kernel test robot <[email protected]>Signed-off-by Rao Shoaib <[email protected]>--- drivers/infiniband/sw/rxe/rxe_param.h | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-)drivers/infiniband/sw/rxe/rxe_param.h +1
-0@@ -9,6 +9,8 @@#include <uapi/rdma/rdma_user_rxe.h> +#define DEFAULT_MAX_VALUE (1 << 20)Can you let me know the link in which the above value is discussed?
Thanks,
Zhu Yanjundrivers/infiniband/sw/rxe/rxe_param.h +13
-13+static inline enum ib_mtu rxe_mtu_int_to_enum (int mtu) {if (mtu <256 )@@ -37,7 +39,7 @@ static inline enum ib_mtu eth_mtu_int_to_enum(int mtu)enum rxe_device_param {RXE_MAX_MR_SIZE =-1ull ,RXE_PAGE_SIZE_CAP =0xfffff000 , -RXE_MAX_QP_WR =0x4000 , +RXE_MAX_QP_WR =DEFAULT_MAX_VALUE ,RXE_DEVICE_CAP_FLAGS =IB_DEVICE_BAD_PKEY_CNTR |IB_DEVICE_BAD_QKEY_CNTR |IB_DEVICE_AUTO_PATH_MIG @@ -58,40 +60,40 @@ enum rxe_device_param {RXE_MAX_INLINE_DATA =RXE_MAX_WQE_SIZE -sizeof (struct rxe_send_wqe ),RXE_MAX_SGE_RD =32 , -RXE_MAX_CQ =16384 , +RXE_MAX_CQ =DEFAULT_MAX_VALUE ,RXE_MAX_LOG_CQE =15 , -RXE_MAX_PD =0x7ffc , +RXE_MAX_PD =DEFAULT_MAX_VALUE ,RXE_MAX_QP_RD_ATOM =128 ,RXE_MAX_RES_RD_ATOM =0x3f000 ,RXE_MAX_QP_INIT_RD_ATOM =128 ,RXE_MAX_MCAST_GRP =8192 ,RXE_MAX_MCAST_QP_ATTACH =56 ,RXE_MAX_TOT_MCAST_QP_ATTACH =0x70000 , -RXE_MAX_AH =100 , -RXE_MAX_SRQ_WR =0x4000 , +RXE_MAX_AH =DEFAULT_MAX_VALUE , +RXE_MAX_SRQ_WR =DEFAULT_MAX_VALUE ,RXE_MIN_SRQ_WR =1 ,RXE_MAX_SRQ_SGE =27 ,RXE_MIN_SRQ_SGE =1 ,RXE_MAX_FMR_PAGE_LIST_LEN =512 , -RXE_MAX_PKEYS =1 , +RXE_MAX_PKEYS =64 ,RXE_LOCAL_CA_ACK_DELAY =15 , -RXE_MAX_UCONTEXT =512 , +RXE_MAX_UCONTEXT =DEFAULT_MAX_VALUE ,RXE_NUM_PORT =1 , -RXE_MAX_QP =0x10000 , +RXE_MAX_QP =DEFAULT_MAX_VALUE ,RXE_MIN_QP_INDEX =16 , -RXE_MAX_QP_INDEX =0x00020000 , +RXE_MAX_QP_INDEX =0x00040000 , -RXE_MAX_SRQ =0x00001000 , +RXE_MAX_SRQ =DEFAULT_MAX_VALUE ,RXE_MIN_SRQ_INDEX =0x00020001 ,RXE_MAX_SRQ_INDEX =0x00040000 , -RXE_MAX_MR =0x00001000 , +RXE_MAX_MR =DEFAULT_MAX_VALUE ,RXE_MAX_MW =0x00001000 ,RXE_MIN_MR_INDEX =0x00000001 , -RXE_MAX_MR_INDEX =0x00010000 , +RXE_MAX_MR_INDEX =0x00040000 ,RXE_MIN_MW_INDEX =0x00010001 , RXE_MAX_MW_INDEX = 0x00020000, --2.27.0
Thanks for reporting the issue. I was able to reproduce the issue and have fixed it. Kindly try gain.
Regards,
Shoaib
Quoted 223 lines · Shoaib Rao
Your urls are garbled, so I can not make any sense out of them.
[root@ca-dev141 linux]# git remote --v
origin https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git (fetch) origin https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git (push)[root@ca-dev141 linux]# git show
8c1b4316c3fa
fatal: ambiguous argument '8c1b4316c3fa': unknown revision or path not in the working tree.[root@ca-dev141 linux]# git log --grep='RDMA/efa: Split hardware' [root@ca-dev141 linux]#
[root@ca-dev141 linux]# git describe
v5.14-rc1-17-gc06676b5953bYou need to do a little bit more work on your end and tell me which value rxe is complaining about. Also provide a kernel version that you are using as I have provided above.
Shoaib
Quoted 203 lines · Zhu Yanjun
Quoted 59 lines · Shoaib Rao
On 7/15/21 9:07 PM, Zhu Yanjun wrote:
On Fri, Jul 16, 2021 at 12:44 AM Shoaib Rao <[email protected]> wrote:
Following is a link
Or just search for my name in the archive.
Do you see any issues with this value?
After this commit is applied, I confronted the following problem "
[ 639.943561]rdma_rxe: unloaded
[ 679.717143]rdma_rxe: loaded
[ 691.721055]rdma_rxe: not enough indices formax_elem
"
Not sure if this problem is introduced by this commit. Please help to check this problem.
Thanks a lot.Zhu Yanjun
I do not see the issue.
[root@ca-dev66 rxe]# lsmod | grep
rdma_rxe
rdma_rxe 126976 0
ip6_udp_tunnel 16384 1rdma_rxe
udp_tunnel 20480 1rdma_rxe
ib_uverbs 147456 3 rdma_rxe,rdma_ucm,mlx5_ib
ib_core 364544 10 rdma_cm,ib_ipoib,rdma_rxe,rds_rdma,iw_cm,ib_umad,rdma_ucm,ib_uverbs,mlx5_ib,ib_cmI am using gdb to dump the values
(gdb) ptype
RXE_MAX_INLINE_DATA
type = enumrxe_device_param{RXE_MAX_MR_SIZE= -1,RXE_PAGE_SIZE_CAP= 4294963200,RXE_MAX_QP_WR= 1048576,RXE_DEVICE_CAP_FLAGS= 137466362998,RXE_MAX_SGE= 32,RXE_MAX_WQE_SIZE= 720,RXE_MAX_INLINE_DATA= 512,RXE_MAX_SGE_RD= 32,RXE_MAX_CQ= 1048576,RXE_MAX_LOG_CQE= 15,RXE_MAX_PD= 1048576,RXE_MAX_QP_RD_ATOM= 128,RXE_MAX_RES_RD_ATOM= 258048,RXE_MAX_QP_INIT_RD_ATOM= 128,RXE_MAX_MCAST_GRP= 8192,RXE_MAX_MCAST_QP_ATTACH= 56,RXE_MAX_TOT_MCAST_QP_ATTACH= 458752,RXE_MAX_AH= 1048576,RXE_MAX_SRQ_WR= 1048576,RXE_MIN_SRQ_WR= 1,RXE_MAX_SRQ_SGE= 27,RXE_MIN_SRQ_SGE= 1,RXE_MAX_FMR_PAGE_LIST_LEN= 512,RXE_MAX_PKEYS= 64,RXE_LOCAL_CA_ACK_DELAY= 15,RXE_MAX_UCONTEXT= 1048576,RXE_NUM_PORT= 1,RXE_MAX_QP= 1048576,RXE_MIN_QP_INDEX= 16,RXE_MAX_QP_INDEX= 262144,RXE_MAX_SRQ= 1048576,RXE_MIN_SRQ_INDEX= 131073,RXE_MAX_SRQ_INDEX= 262144,RXE_MAX_MR= 1048576,RXE_MAX_MW= 4096,RXE_MIN_MR_INDEX= 1,RXE_MAX_MR_INDEX= 262144,RXE_MIN_MW_INDEX= 65537,RXE_MAX_MW_INDEX= 131072,RXE_MAX_PKT_PER_ACK= 64,RXE_MAX_UNACKED_PSNS= 128,RXE_INFLIGHT_SKBS_PER_QP_HIGH= 64,RXE_INFLIGHT_SKBS_PER_QP_LOW= 16,RXE_NSEC_ARB_TIMER_DELAY= 200,RXE_VENDOR_ID= 16777215}Quoted 1 line
It is possible that you are changing some values.
I git clone the source code from
https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git__;!!ACWV5N9M2RV99hQ!efPsL9fikzm6Ob1-zsmbsAzPkfqRpxYkp9oJXF9csPOJgoei9njj8RmA5IyuH447$ again. And the first 3 commits are as below. One is your commit. " 6163ad3208c8 (HEAD -> for-next) RDMA/rxe: Bump up default maximum values used via uverbs 8c1b4316c3fa (origin/for-next, origin/HEAD) RDMA/efa: Split hardware stats to device and port stats 916071185b17 MAINTAINERS: Update maintainers of HiSilicon RoCE "And when I tried to add a new rxe0,
"
error: Invalid argument
"
And the dmesg logs are as below:# dmesg " [ 70.782302] e1000: enp0s8 NIC Link is Up 1000 Mbps Full Duplex, Flow Control: RX [ 70.782744] IPv6: ADDRCONF(NETDEV_CHANGE): enp0s8: link becomes ready [ 79.467652] rdma_rxe: loaded [ 79.468348] rdma_rxe: not enough indices for max_elem [ 79.468356] rdma_rxe: failed to add enp0s8 "Please follow my steps to reproduce this problem again.
Zhu Yanjun
Quoted 110 lines
Shoaib
Shoaib
On 7/14/21 10:02 PM, Zhu Yanjun wrote:
On Wed, Jul 14, 2021 at 4:36 PM Rao Shoaib <[email protected]> wrote:
From: Rao Shoaib <[email protected]>
In our internal testing we have found that the current maximum are too smalls. Ideally there should be no limits but currently maximum values are reported via
ibv_query_device, so we have to keep maximum values but they have been made suffiently large.Resubmitting after fixing an issue reported by test robot.
Reported-by kernel test robot <[email protected]>Signed-off-by Rao Shoaib <[email protected]>---
drivers/infiniband/sw/rxe/rxe_param.h| 26
++++++++++++++------------1 file changed, 14 insertions(+), 12 deletions(-)drivers/infiniband/sw/rxe/rxe_param.h +1
-0@@ -9,6 +9,8 @@ #include <uapi/rdma/rdma_user_rxe.h> +#define DEFAULT_MAX_VALUE (1 << 20)Can you let me know the link in which the above value is discussed?
Thanks,
Zhu Yanjun+
static inline enumib_mturxe_mtu_int_to_enum(int mtu){
if (mtu < 256)drivers/infiniband/sw/rxe/rxe_param.h +1
-1@@ -37,7 +39,7 @@ static inline enum ib_mtu eth_mtu_int_to_enum(int mtu) enum rxe_device_param { RXE_MAX_MR_SIZE = -1ull, RXE_PAGE_SIZE_CAP = 0xfffff000, - RXE_MAX_QP_WR = 0x4000, + RXE_MAX_QP_WR = DEFAULT_MAX_VALUE,RXE_DEVICE_CAP_FLAGS =
IB_DEVICE_BAD_PKEY_CNTR|IB_DEVICE_BAD_QKEY_CNTR|IB_DEVICE_AUTO_PATH_MIGdrivers/infiniband/sw/rxe/rxe_param.h +0
-0@@ -58,40 +60,40 @@ enum rxe_device_param {RXE_MAX_INLINE_DATA =
RXE_MAX_WQE_SIZE-sizeof(struct rxe_send_wqe), RXE_MAX_SGE_RD = 32,
- RXE_MAX_CQ = 16384,
+ RXE_MAX_CQ =DEFAULT_MAX_VALUE, RXE_MAX_LOG_CQE = 15,
- RXE_MAX_PD =0x7ffc,
+ RXE_MAX_PD =DEFAULT_MAX_VALUE, RXE_MAX_QP_RD_ATOM = 128, RXE_MAX_RES_RD_ATOM =0x3f000, RXE_MAX_QP_INIT_RD_ATOM = 128, RXE_MAX_MCAST_GRP = 8192, RXE_MAX_MCAST_QP_ATTACH = 56, RXE_MAX_TOT_MCAST_QP_ATTACH =0x70000,
- RXE_MAX_AH = 100,
- RXE_MAX_SRQ_WR =0x4000,
+ RXE_MAX_AH =DEFAULT_MAX_VALUE, + RXE_MAX_SRQ_WR =DEFAULT_MAX_VALUE, RXE_MIN_SRQ_WR = 1, RXE_MAX_SRQ_SGE = 27, RXE_MIN_SRQ_SGE = 1, RXE_MAX_FMR_PAGE_LIST_LEN = 512,
- RXE_MAX_PKEYS = 1,
+ RXE_MAX_PKEYS = 64, RXE_LOCAL_CA_ACK_DELAY = 15,- RXE_MAX_UCONTEXT = 512,
+ RXE_MAX_UCONTEXT =DEFAULT_MAX_VALUE,RXE_NUM_PORT = 1,
- RXE_MAX_QP =
0x10000, + RXE_MAX_QP =DEFAULT_MAX_VALUE, RXE_MIN_QP_INDEX = 16,
- RXE_MAX_QP_INDEX =0x00020000, + RXE_MAX_QP_INDEX =0x00040000,- RXE_MAX_SRQ =
0x00001000, + RXE_MAX_SRQ =DEFAULT_MAX_VALUE, RXE_MIN_SRQ_INDEX =0x00020001, RXE_MAX_SRQ_INDEX =0x00040000,- RXE_MAX_MR =
0x00001000, + RXE_MAX_MR =DEFAULT_MAX_VALUE, RXE_MAX_MW =0x00001000, RXE_MIN_MR_INDEX =0x00000001, - RXE_MAX_MR_INDEX =0x00010000, + RXE_MAX_MR_INDEX =0x00040000, RXE_MIN_MW_INDEX =0x00010001, RXE_MAX_MW_INDEX =0x00020000,Signature · 1 line
2.27.0
Zhu Yanjun
maintainerQuoted 3 lines · Shoaib Rao
Thanks for reporting the issue. I was able to reproduce the issue and have fixed it. Kindly try gain.
Do you make tests with rdma-core?
IMHO, it is important to make tests with rdma-core before the commit is sent to the community.
Zhu Yanjun
Quoted 250 lines
Regards,
Shoaib
Quoted 243 lines · Shoaib Rao
Your urls are garbled, so I can not make any sense out of them.
[root@ca-dev141 linux]# git remote --v
origin
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
(fetch)
origin
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git (push)[root@ca-dev141 linux]# git show
8c1b4316c3fa
fatal: ambiguous argument '8c1b4316c3fa': unknown revision or path not in the working tree.[root@ca-dev141 linux]# git log --grep='RDMA/efa: Split hardware' [root@ca-dev141 linux]#
[root@ca-dev141 linux]# git describe
v5.14-rc1-17-gc06676b5953bYou need to do a little bit more work on your end and tell me which value rxe is complaining about. Also provide a kernel version that you are using as I have provided above.
Shoaib
Quoted 217 lines · Zhu Yanjun
Quoted 68 lines · Shoaib Rao
On 7/15/21 9:07 PM, Zhu Yanjun wrote:
On Fri, Jul 16, 2021 at 12:44 AM Shoaib Rao <[email protected]> wrote:
Following is a link
Or just search for my name in the archive.
Do you see any issues with this value?
After this commit is applied, I confronted the following problem "
[ 639.943561] rdma_rxe: unloaded [ 679.717143] rdma_rxe: loaded [ 691.721055] rdma_rxe: not enough indices for max_elem"
Not sure if this problem is introduced by this commit. Please help to check this problem.
Thanks a lot.Zhu Yanjun
I do not see the issue.
[root@ca-dev66 rxe]# lsmod | grep
rdma_rxerdma_rxe 126976 0 ip6_udp_tunnel 16384 1 rdma_rxe udp_tunnel 20480 1 rdma_rxe ib_uverbs 147456 3 rdma_rxe,rdma_ucm,mlx5_ib ib_core 364544 10rdma_cm,ib_ipoib,rdma_rxe,rds_rdma,iw_cm,ib_umad,rdma_ucm,ib_uverbs,mlx5_ib,ib_cm
I am using gdb to dump the values
(gdb) ptype
RXE_MAX_INLINE_DATA
type = enumrxe_device_param{RXE_MAX_MR_SIZE= -1,RXE_PAGE_SIZE_CAP = 4294963200, RXE_MAX_QP_WR = 1048576, RXE_DEVICE_CAP_FLAGS = 137466362998, RXE_MAX_SGE = 32, RXE_MAX_WQE_SIZE = 720, RXE_MAX_INLINE_DATA = 512, RXE_MAX_SGE_RD = 32, RXE_MAX_CQ = 1048576, RXE_MAX_LOG_CQE = 15, RXE_MAX_PD = 1048576, RXE_MAX_QP_RD_ATOM = 128, RXE_MAX_RES_RD_ATOM = 258048, RXE_MAX_QP_INIT_RD_ATOM = 128, RXE_MAX_MCAST_GRP = 8192, RXE_MAX_MCAST_QP_ATTACH = 56, RXE_MAX_TOT_MCAST_QP_ATTACH = 458752, RXE_MAX_AH = 1048576, RXE_MAX_SRQ_WR = 1048576, RXE_MIN_SRQ_WR = 1, RXE_MAX_SRQ_SGE = 27, RXE_MIN_SRQ_SGE = 1, RXE_MAX_FMR_PAGE_LIST_LEN = 512, RXE_MAX_PKEYS = 64, RXE_LOCAL_CA_ACK_DELAY = 15, RXE_MAX_UCONTEXT = 1048576, RXE_NUM_PORT = 1, RXE_MAX_QP = 1048576, RXE_MIN_QP_INDEX = 16, RXE_MAX_QP_INDEX = 262144, RXE_MAX_SRQ = 1048576, RXE_MIN_SRQ_INDEX = 131073, RXE_MAX_SRQ_INDEX = 262144, RXE_MAX_MR = 1048576, RXE_MAX_MW = 4096, RXE_MIN_MR_INDEX = 1, RXE_MAX_MR_INDEX = 262144, RXE_MIN_MW_INDEX = 65537, RXE_MAX_MW_INDEX = 131072, RXE_MAX_PKT_PER_ACK = 64, RXE_MAX_UNACKED_PSNS = 128, RXE_INFLIGHT_SKBS_PER_QP_HIGH = 64, RXE_INFLIGHT_SKBS_PER_QP_LOW = 16, RXE_NSEC_ARB_TIMER_DELAY = 200, RXE_VENDOR_ID = 16777215}Quoted 1 line
It is possible that you are changing some values.
I git clone the source code from
https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git__;!!ACWV5N9M2RV99hQ!efPsL9fikzm6Ob1-zsmbsAzPkfqRpxYkp9oJXF9csPOJgoei9njj8RmA5IyuH447$
again.
And the first 3 commits are as below. One is your commit. "6163ad3208c8(HEAD -> for-next) RDMA/rxe: Bump up default maximum values used via uverbs8c1b4316c3fa(origin/for-next, origin/HEAD) RDMA/efa: Split hardware stats to device and port stats916071185b17MAINTAINERS: Update maintainers of HiSilicon RoCE "And when I tried to add a new rxe0,
"
error: Invalid argument
"
And the dmesg logs are as below:# dmesg " [ 70.782302] e1000: enp0s8 NIC Link is Up 1000 Mbps Full Duplex, Flow Control: RX [ 70.782744] IPv6: ADDRCONF(NETDEV_CHANGE): enp0s8: link becomes ready [ 79.467652] rdma_rxe: loaded [ 79.468348] rdma_rxe: not enough indices for max_elem [ 79.468356] rdma_rxe: failed to add enp0s8 "Please follow my steps to reproduce this problem again.
Zhu Yanjun
Quoted 113 lines
Shoaib
Shoaib
On 7/14/21 10:02 PM, Zhu Yanjun wrote:
On Wed, Jul 14, 2021 at 4:36 PM Rao Shoaib <[email protected]> wrote:
From: Rao Shoaib <[email protected]>
In our internal testing we have found that the current maximum are too smalls. Ideally there should be no limits but currently maximum values are reported via
ibv_query_device, so we have to keep maximum values but they have been made suffiently large.Resubmitting after fixing an issue reported by test robot.
Reported-by kernel test robot <[email protected]>Signed-off-by Rao Shoaib <[email protected]>--- drivers/infiniband/sw/rxe/rxe_param.h | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-)drivers/infiniband/sw/rxe/rxe_param.h +1
-0@@ -9,6 +9,8 @@#include <uapi/rdma/rdma_user_rxe.h> +#define DEFAULT_MAX_VALUE (1 << 20)Can you let me know the link in which the above value is discussed?
Thanks,
Zhu Yanjundrivers/infiniband/sw/rxe/rxe_param.h +2
-1+static inline enum ib_mtu rxe_mtu_int_to_enum (int mtu) {if (mtu <256 )@@ -37,7 +39,7 @@ static inline enum ib_mtu eth_mtu_int_to_enum(int mtu)enum rxe_device_param {RXE_MAX_MR_SIZE =-1ull ,RXE_PAGE_SIZE_CAP =0xfffff000 , -RXE_MAX_QP_WR =0x4000 , +RXE_MAX_QP_WR =DEFAULT_MAX_VALUE ,RXE_DEVICE_CAP_FLAGS =IB_DEVICE_BAD_PKEY_CNTR |IB_DEVICE_BAD_QKEY_CNTR |
IB_DEVICE_AUTO_PATH_MIGdrivers/infiniband/sw/rxe/rxe_param.h +11
-12@@ -58,40 +60,40 @@ enum rxe_device_param {RXE_MAX_INLINE_DATA =RXE_MAX_WQE_SIZE -sizeof (struct rxe_send_wqe ),RXE_MAX_SGE_RD =32 , -RXE_MAX_CQ =16384 , +RXE_MAX_CQ =DEFAULT_MAX_VALUE ,RXE_MAX_LOG_CQE =15 , -RXE_MAX_PD =0x7ffc , +RXE_MAX_PD =DEFAULT_MAX_VALUE ,RXE_MAX_QP_RD_ATOM =128 ,RXE_MAX_RES_RD_ATOM =0x3f000 ,RXE_MAX_QP_INIT_RD_ATOM =128 ,RXE_MAX_MCAST_GRP =8192 ,RXE_MAX_MCAST_QP_ATTACH =56 ,RXE_MAX_TOT_MCAST_QP_ATTACH =0x70000 , -RXE_MAX_AH =100 , -RXE_MAX_SRQ_WR =0x4000 , +RXE_MAX_AH =DEFAULT_MAX_VALUE , +RXE_MAX_SRQ_WR =DEFAULT_MAX_VALUE ,RXE_MIN_SRQ_WR =1 ,RXE_MAX_SRQ_SGE =27 ,RXE_MIN_SRQ_SGE =1 ,RXE_MAX_FMR_PAGE_LIST_LEN =512 , -RXE_MAX_PKEYS =1 , +RXE_MAX_PKEYS =64 ,RXE_LOCAL_CA_ACK_DELAY =15 , -RXE_MAX_UCONTEXT =512 , +RXE_MAX_UCONTEXT =DEFAULT_MAX_VALUE ,RXE_NUM_PORT =1 , -RXE_MAX_QP =0x10000 , +RXE_MAX_QP =DEFAULT_MAX_VALUE ,RXE_MIN_QP_INDEX =16 , -RXE_MAX_QP_INDEX =0x00020000 , +RXE_MAX_QP_INDEX =0x00040000 , -RXE_MAX_SRQ =0x00001000 , +RXE_MAX_SRQ =DEFAULT_MAX_VALUE ,RXE_MIN_SRQ_INDEX =0x00020001 ,RXE_MAX_SRQ_INDEX =0x00040000 , -RXE_MAX_MR =0x00001000 , +RXE_MAX_MR =DEFAULT_MAX_VALUE ,RXE_MAX_MW =0x00001000 ,RXE_MIN_MR_INDEX =0x00000001 , -RXE_MAX_MR_INDEX =0x00010000 , +RXE_MAX_MR_INDEX =0x00040000 ,RXE_MIN_MW_INDEX =0x00010001 , RXE_MAX_MW_INDEX = 0x00020000, --2.27.0
Quoted 7 lines · Zhu Yanjun
Quoted 2 lines · Shoaib Rao
Thanks for reporting the issue. I was able to reproduce the issue and have fixed it. Kindly try gain.
Do you make tests with rdma-core?
IMHO, it is important to make tests with rdma-core before the commit is sent to the community.
Yes I did. However the issue shows up only when I configure an interface not just when module is loading. Anyways, my apologies.
Shoaib
Quoted 250 lines
Zhu Yanjun
Quoted 247 lines
Regards,
Shoaib
Quoted 241 lines · Shoaib Rao
Your urls are garbled, so I can not make any sense out of them.
[root@ca-dev141 linux]# git remote --v
origin
https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git__;!!ACWV5N9M2RV99hQ!dGmB_LFQF6at31F0sVXbfyjtAlgNTWzBEaIER13mVG0ZAWBomLdmWXUe6DxoVFcz$
(fetch)
originhttps://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git__;!!ACWV5N9M2RV99hQ!dGmB_LFQF6at31F0sVXbfyjtAlgNTWzBEaIER13mVG0ZAWBomLdmWXUe6DxoVFcz$ (push)[root@ca-dev141 linux]# git show
8c1b4316c3fa
fatal: ambiguous argument '8c1b4316c3fa': unknown revision or path not in the working tree.[root@ca-dev141 linux]# git log --grep='RDMA/efa: Split hardware' [root@ca-dev141 linux]#
[root@ca-dev141 linux]# git describe
v5.14-rc1-17-gc06676b5953bYou need to do a little bit more work on your end and tell me which value rxe is complaining about. Also provide a kernel version that you are using as I have provided above.
Shoaib
Quoted 215 lines · Zhu Yanjun
Quoted 67 lines · Shoaib Rao
On 7/15/21 9:07 PM, Zhu Yanjun wrote:
On Fri, Jul 16, 2021 at 12:44 AM Shoaib Rao <[email protected]> wrote:
Following is a link
Or just search for my name in the archive.
Do you see any issues with this value?
After this commit is applied, I confronted the following problem "
[ 639.943561] rdma_rxe: unloaded [ 679.717143] rdma_rxe: loaded [ 691.721055] rdma_rxe: not enough indices for max_elem"
Not sure if this problem is introduced by this commit. Please help to check this problem.
Thanks a lot.Zhu Yanjun
I do not see the issue.
[root@ca-dev66 rxe]# lsmod | grep
rdma_rxerdma_rxe 126976 0 ip6_udp_tunnel 16384 1 rdma_rxe udp_tunnel 20480 1 rdma_rxe ib_uverbs 147456 3 rdma_rxe,rdma_ucm,mlx5_ib ib_core 364544 10rdma_cm,ib_ipoib,rdma_rxe,rds_rdma,iw_cm,ib_umad,rdma_ucm,ib_uverbs,mlx5_ib,ib_cm
I am using gdb to dump the values
(gdb) ptype
RXE_MAX_INLINE_DATA
type = enumrxe_device_param{RXE_MAX_MR_SIZE= -1,RXE_PAGE_SIZE_CAP = 4294963200, RXE_MAX_QP_WR = 1048576, RXE_DEVICE_CAP_FLAGS = 137466362998, RXE_MAX_SGE = 32, RXE_MAX_WQE_SIZE = 720, RXE_MAX_INLINE_DATA = 512, RXE_MAX_SGE_RD = 32, RXE_MAX_CQ = 1048576, RXE_MAX_LOG_CQE = 15, RXE_MAX_PD = 1048576, RXE_MAX_QP_RD_ATOM = 128, RXE_MAX_RES_RD_ATOM = 258048, RXE_MAX_QP_INIT_RD_ATOM = 128, RXE_MAX_MCAST_GRP = 8192, RXE_MAX_MCAST_QP_ATTACH = 56, RXE_MAX_TOT_MCAST_QP_ATTACH = 458752, RXE_MAX_AH = 1048576, RXE_MAX_SRQ_WR = 1048576, RXE_MIN_SRQ_WR = 1, RXE_MAX_SRQ_SGE = 27, RXE_MIN_SRQ_SGE = 1, RXE_MAX_FMR_PAGE_LIST_LEN = 512, RXE_MAX_PKEYS = 64, RXE_LOCAL_CA_ACK_DELAY = 15, RXE_MAX_UCONTEXT = 1048576, RXE_NUM_PORT = 1, RXE_MAX_QP = 1048576, RXE_MIN_QP_INDEX = 16, RXE_MAX_QP_INDEX = 262144, RXE_MAX_SRQ = 1048576, RXE_MIN_SRQ_INDEX = 131073, RXE_MAX_SRQ_INDEX = 262144, RXE_MAX_MR = 1048576, RXE_MAX_MW = 4096, RXE_MIN_MR_INDEX = 1, RXE_MAX_MR_INDEX = 262144, RXE_MIN_MW_INDEX = 65537, RXE_MAX_MW_INDEX = 131072, RXE_MAX_PKT_PER_ACK = 64, RXE_MAX_UNACKED_PSNS = 128, RXE_INFLIGHT_SKBS_PER_QP_HIGH = 64, RXE_INFLIGHT_SKBS_PER_QP_LOW = 16, RXE_NSEC_ARB_TIMER_DELAY = 200, RXE_VENDOR_ID = 16777215}Quoted 1 line
It is possible that you are changing some values.
I git clone the source code from
https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git__;!!ACWV5N9M2RV99hQ!efPsL9fikzm6Ob1-zsmbsAzPkfqRpxYkp9oJXF9csPOJgoei9njj8RmA5IyuH447$
again.
And the first 3 commits are as below. One is your commit. "6163ad3208c8(HEAD -> for-next) RDMA/rxe: Bump up default maximum values used via uverbs8c1b4316c3fa(origin/for-next, origin/HEAD) RDMA/efa: Split hardware stats to device and port stats916071185b17MAINTAINERS: Update maintainers of HiSilicon RoCE "And when I tried to add a new rxe0,
"
error: Invalid argument
"
And the dmesg logs are as below:# dmesg " [ 70.782302] e1000: enp0s8 NIC Link is Up 1000 Mbps Full Duplex, Flow Control: RX [ 70.782744] IPv6: ADDRCONF(NETDEV_CHANGE): enp0s8: link becomes ready [ 79.467652] rdma_rxe: loaded [ 79.468348] rdma_rxe: not enough indices for max_elem [ 79.468356] rdma_rxe: failed to add enp0s8 "Please follow my steps to reproduce this problem again.
Zhu Yanjun
Quoted 113 lines
Shoaib
Shoaib
On 7/14/21 10:02 PM, Zhu Yanjun wrote:
On Wed, Jul 14, 2021 at 4:36 PM Rao Shoaib <[email protected]> wrote:
From: Rao Shoaib <[email protected]>
In our internal testing we have found that the current maximum are too smalls. Ideally there should be no limits but currently maximum values are reported via
ibv_query_device, so we have to keep maximum values but they have been made suffiently large.Resubmitting after fixing an issue reported by test robot.
Reported-by kernel test robot <[email protected]>Signed-off-by Rao Shoaib <[email protected]>--- drivers/infiniband/sw/rxe/rxe_param.h | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-)drivers/infiniband/sw/rxe/rxe_param.h +1
-0@@ -9,6 +9,8 @@#include <uapi/rdma/rdma_user_rxe.h> +#define DEFAULT_MAX_VALUE (1 << 20)Can you let me know the link in which the above value is discussed?
Thanks,
Zhu Yanjundrivers/infiniband/sw/rxe/rxe_param.h +2
-1+static inline enum ib_mtu rxe_mtu_int_to_enum (int mtu) {if (mtu <256 )@@ -37,7 +39,7 @@ static inline enum ib_mtu eth_mtu_int_to_enum(int mtu)enum rxe_device_param {RXE_MAX_MR_SIZE =-1ull ,RXE_PAGE_SIZE_CAP =0xfffff000 , -RXE_MAX_QP_WR =0x4000 , +RXE_MAX_QP_WR =DEFAULT_MAX_VALUE ,RXE_DEVICE_CAP_FLAGS =IB_DEVICE_BAD_PKEY_CNTR |IB_DEVICE_BAD_QKEY_CNTR |
IB_DEVICE_AUTO_PATH_MIGdrivers/infiniband/sw/rxe/rxe_param.h +11
-12@@ -58,40 +60,40 @@ enum rxe_device_param {RXE_MAX_INLINE_DATA =RXE_MAX_WQE_SIZE -sizeof (struct rxe_send_wqe ),RXE_MAX_SGE_RD =32 , -RXE_MAX_CQ =16384 , +RXE_MAX_CQ =DEFAULT_MAX_VALUE ,RXE_MAX_LOG_CQE =15 , -RXE_MAX_PD =0x7ffc , +RXE_MAX_PD =DEFAULT_MAX_VALUE ,RXE_MAX_QP_RD_ATOM =128 ,RXE_MAX_RES_RD_ATOM =0x3f000 ,RXE_MAX_QP_INIT_RD_ATOM =128 ,RXE_MAX_MCAST_GRP =8192 ,RXE_MAX_MCAST_QP_ATTACH =56 ,RXE_MAX_TOT_MCAST_QP_ATTACH =0x70000 , -RXE_MAX_AH =100 , -RXE_MAX_SRQ_WR =0x4000 , +RXE_MAX_AH =DEFAULT_MAX_VALUE , +RXE_MAX_SRQ_WR =DEFAULT_MAX_VALUE ,RXE_MIN_SRQ_WR =1 ,RXE_MAX_SRQ_SGE =27 ,RXE_MIN_SRQ_SGE =1 ,RXE_MAX_FMR_PAGE_LIST_LEN =512 , -RXE_MAX_PKEYS =1 , +RXE_MAX_PKEYS =64 ,RXE_LOCAL_CA_ACK_DELAY =15 , -RXE_MAX_UCONTEXT =512 , +RXE_MAX_UCONTEXT =DEFAULT_MAX_VALUE ,RXE_NUM_PORT =1 , -RXE_MAX_QP =0x10000 , +RXE_MAX_QP =DEFAULT_MAX_VALUE ,RXE_MIN_QP_INDEX =16 , -RXE_MAX_QP_INDEX =0x00020000 , +RXE_MAX_QP_INDEX =0x00040000 , -RXE_MAX_SRQ =0x00001000 , +RXE_MAX_SRQ =DEFAULT_MAX_VALUE ,RXE_MIN_SRQ_INDEX =0x00020001 ,RXE_MAX_SRQ_INDEX =0x00040000 , -RXE_MAX_MR =0x00001000 , +RXE_MAX_MR =DEFAULT_MAX_VALUE ,RXE_MAX_MW =0x00001000 ,RXE_MIN_MR_INDEX =0x00000001 , -RXE_MAX_MR_INDEX =0x00010000 , +RXE_MAX_MR_INDEX =0x00040000 ,RXE_MIN_MW_INDEX =0x00010001 , RXE_MAX_MW_INDEX = 0x00020000, --2.27.0
Zhu Yanjun
maintainerQuoted 6 lines · Shoaib Rao
Quoted 3 lines · Zhu Yanjun
Quoted 2 lines · Shoaib Rao
Thanks for reporting the issue. I was able to reproduce the issue and have fixed it. Kindly try gain.
Quoted 7 lines
Quoted 4 lines
Do you make tests with rdma-core?
IMHO, it is important to make tests with rdma-core before the commit is sent to the community.
Yes I did. However the issue shows up only when I configure an interface not just when module is loading. Anyways, my apologies.
Thanks.
In my test environment, it can pass rdma-core tests. But in this commit, a lot of default maximum values are changed. Not sure if this will cause other issues.
Please wait for others' comments.
Thank you
Zhu Yanjun
Quoted 253 lines
Shoaib
Quoted 250 lines
Zhu Yanjun
Quoted 247 lines
Regards,
Shoaib
Quoted 241 lines · Shoaib Rao
Your urls are garbled, so I can not make any sense out of them.
[root@ca-dev141 linux]# git remote --v
origin
https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git__;!!ACWV5N9M2RV99hQ!dGmB_LFQF6at31F0sVXbfyjtAlgNTWzBEaIER13mVG0ZAWBomLdmWXUe6DxoVFcz$
(fetch)
originhttps://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git__;!!ACWV5N9M2RV99hQ!dGmB_LFQF6at31F0sVXbfyjtAlgNTWzBEaIER13mVG0ZAWBomLdmWXUe6DxoVFcz$ (push)[root@ca-dev141 linux]# git show
8c1b4316c3fa
fatal: ambiguous argument '8c1b4316c3fa': unknown revision or path not in the working tree.[root@ca-dev141 linux]# git log --grep='RDMA/efa: Split hardware' [root@ca-dev141 linux]#
[root@ca-dev141 linux]# git describe
v5.14-rc1-17-gc06676b5953bYou need to do a little bit more work on your end and tell me which value rxe is complaining about. Also provide a kernel version that you are using as I have provided above.
Shoaib
Quoted 215 lines · Zhu Yanjun
Quoted 67 lines · Shoaib Rao
On 7/15/21 9:07 PM, Zhu Yanjun wrote:
On Fri, Jul 16, 2021 at 12:44 AM Shoaib Rao <[email protected]> wrote:
Following is a link
Or just search for my name in the archive.
Do you see any issues with this value?
After this commit is applied, I confronted the following problem "
[ 639.943561] rdma_rxe: unloaded [ 679.717143] rdma_rxe: loaded [ 691.721055] rdma_rxe: not enough indices for max_elem"
Not sure if this problem is introduced by this commit. Please help to check this problem.
Thanks a lot.Zhu Yanjun
I do not see the issue.
[root@ca-dev66 rxe]# lsmod | grep
rdma_rxerdma_rxe 126976 0 ip6_udp_tunnel 16384 1 rdma_rxe udp_tunnel 20480 1 rdma_rxe ib_uverbs 147456 3 rdma_rxe,rdma_ucm,mlx5_ib ib_core 364544 10rdma_cm,ib_ipoib,rdma_rxe,rds_rdma,iw_cm,ib_umad,rdma_ucm,ib_uverbs,mlx5_ib,ib_cm
I am using gdb to dump the values
(gdb) ptype
RXE_MAX_INLINE_DATA
type = enumrxe_device_param{RXE_MAX_MR_SIZE= -1,RXE_PAGE_SIZE_CAP = 4294963200, RXE_MAX_QP_WR = 1048576, RXE_DEVICE_CAP_FLAGS = 137466362998, RXE_MAX_SGE = 32, RXE_MAX_WQE_SIZE = 720, RXE_MAX_INLINE_DATA = 512, RXE_MAX_SGE_RD = 32, RXE_MAX_CQ = 1048576, RXE_MAX_LOG_CQE = 15, RXE_MAX_PD = 1048576, RXE_MAX_QP_RD_ATOM = 128, RXE_MAX_RES_RD_ATOM = 258048, RXE_MAX_QP_INIT_RD_ATOM = 128, RXE_MAX_MCAST_GRP = 8192, RXE_MAX_MCAST_QP_ATTACH = 56, RXE_MAX_TOT_MCAST_QP_ATTACH = 458752, RXE_MAX_AH = 1048576, RXE_MAX_SRQ_WR = 1048576, RXE_MIN_SRQ_WR = 1, RXE_MAX_SRQ_SGE = 27, RXE_MIN_SRQ_SGE = 1, RXE_MAX_FMR_PAGE_LIST_LEN = 512, RXE_MAX_PKEYS = 64, RXE_LOCAL_CA_ACK_DELAY = 15, RXE_MAX_UCONTEXT = 1048576, RXE_NUM_PORT = 1, RXE_MAX_QP = 1048576, RXE_MIN_QP_INDEX = 16, RXE_MAX_QP_INDEX = 262144, RXE_MAX_SRQ = 1048576, RXE_MIN_SRQ_INDEX = 131073, RXE_MAX_SRQ_INDEX = 262144, RXE_MAX_MR = 1048576, RXE_MAX_MW = 4096, RXE_MIN_MR_INDEX = 1, RXE_MAX_MR_INDEX = 262144, RXE_MIN_MW_INDEX = 65537, RXE_MAX_MW_INDEX = 131072, RXE_MAX_PKT_PER_ACK = 64, RXE_MAX_UNACKED_PSNS = 128, RXE_INFLIGHT_SKBS_PER_QP_HIGH = 64, RXE_INFLIGHT_SKBS_PER_QP_LOW = 16, RXE_NSEC_ARB_TIMER_DELAY = 200, RXE_VENDOR_ID = 16777215}Quoted 1 line
It is possible that you are changing some values.
I git clone the source code from
https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git__;!!ACWV5N9M2RV99hQ!efPsL9fikzm6Ob1-zsmbsAzPkfqRpxYkp9oJXF9csPOJgoei9njj8RmA5IyuH447$
again.
And the first 3 commits are as below. One is your commit. "6163ad3208c8(HEAD -> for-next) RDMA/rxe: Bump up default maximum values used via uverbs8c1b4316c3fa(origin/for-next, origin/HEAD) RDMA/efa: Split hardware stats to device and port stats916071185b17MAINTAINERS: Update maintainers of HiSilicon RoCE "And when I tried to add a new rxe0,
"
error: Invalid argument
"
And the dmesg logs are as below:# dmesg " [ 70.782302] e1000: enp0s8 NIC Link is Up 1000 Mbps Full Duplex, Flow Control: RX [ 70.782744] IPv6: ADDRCONF(NETDEV_CHANGE): enp0s8: link becomes ready [ 79.467652] rdma_rxe: loaded [ 79.468348] rdma_rxe: not enough indices for max_elem [ 79.468356] rdma_rxe: failed to add enp0s8 "Please follow my steps to reproduce this problem again.
Zhu Yanjun
Quoted 113 lines
Shoaib
Shoaib
On 7/14/21 10:02 PM, Zhu Yanjun wrote:
On Wed, Jul 14, 2021 at 4:36 PM Rao Shoaib <[email protected]> wrote:
From: Rao Shoaib <[email protected]>
In our internal testing we have found that the current maximum are too smalls. Ideally there should be no limits but currently maximum values are reported via
ibv_query_device, so we have to keep maximum values but they have been made suffiently large.Resubmitting after fixing an issue reported by test robot.
Reported-by kernel test robot <[email protected]>Signed-off-by Rao Shoaib <[email protected]>--- drivers/infiniband/sw/rxe/rxe_param.h | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-)drivers/infiniband/sw/rxe/rxe_param.h +1
-0@@ -9,6 +9,8 @@#include <uapi/rdma/rdma_user_rxe.h> +#define DEFAULT_MAX_VALUE (1 << 20)Can you let me know the link in which the above value is discussed?
Thanks,
Zhu Yanjundrivers/infiniband/sw/rxe/rxe_param.h +2
-1+static inline enum ib_mtu rxe_mtu_int_to_enum (int mtu) {if (mtu <256 )@@ -37,7 +39,7 @@ static inline enum ib_mtu eth_mtu_int_to_enum(int mtu)enum rxe_device_param {RXE_MAX_MR_SIZE =-1ull ,RXE_PAGE_SIZE_CAP =0xfffff000 , -RXE_MAX_QP_WR =0x4000 , +RXE_MAX_QP_WR =DEFAULT_MAX_VALUE ,RXE_DEVICE_CAP_FLAGS =IB_DEVICE_BAD_PKEY_CNTR |IB_DEVICE_BAD_QKEY_CNTR |
IB_DEVICE_AUTO_PATH_MIGdrivers/infiniband/sw/rxe/rxe_param.h +11
-12@@ -58,40 +60,40 @@ enum rxe_device_param {RXE_MAX_INLINE_DATA =RXE_MAX_WQE_SIZE -sizeof (struct rxe_send_wqe ),RXE_MAX_SGE_RD =32 , -RXE_MAX_CQ =16384 , +RXE_MAX_CQ =DEFAULT_MAX_VALUE ,RXE_MAX_LOG_CQE =15 , -RXE_MAX_PD =0x7ffc , +RXE_MAX_PD =DEFAULT_MAX_VALUE ,RXE_MAX_QP_RD_ATOM =128 ,RXE_MAX_RES_RD_ATOM =0x3f000 ,RXE_MAX_QP_INIT_RD_ATOM =128 ,RXE_MAX_MCAST_GRP =8192 ,RXE_MAX_MCAST_QP_ATTACH =56 ,RXE_MAX_TOT_MCAST_QP_ATTACH =0x70000 , -RXE_MAX_AH =100 , -RXE_MAX_SRQ_WR =0x4000 , +RXE_MAX_AH =DEFAULT_MAX_VALUE , +RXE_MAX_SRQ_WR =DEFAULT_MAX_VALUE ,RXE_MIN_SRQ_WR =1 ,RXE_MAX_SRQ_SGE =27 ,RXE_MIN_SRQ_SGE =1 ,RXE_MAX_FMR_PAGE_LIST_LEN =512 , -RXE_MAX_PKEYS =1 , +RXE_MAX_PKEYS =64 ,RXE_LOCAL_CA_ACK_DELAY =15 , -RXE_MAX_UCONTEXT =512 , +RXE_MAX_UCONTEXT =DEFAULT_MAX_VALUE ,RXE_NUM_PORT =1 , -RXE_MAX_QP =0x10000 , +RXE_MAX_QP =DEFAULT_MAX_VALUE ,RXE_MIN_QP_INDEX =16 , -RXE_MAX_QP_INDEX =0x00020000 , +RXE_MAX_QP_INDEX =0x00040000 , -RXE_MAX_SRQ =0x00001000 , +RXE_MAX_SRQ =DEFAULT_MAX_VALUE ,RXE_MIN_SRQ_INDEX =0x00020001 ,RXE_MAX_SRQ_INDEX =0x00040000 , -RXE_MAX_MR =0x00001000 , +RXE_MAX_MR =DEFAULT_MAX_VALUE ,RXE_MAX_MW =0x00001000 ,RXE_MIN_MR_INDEX =0x00000001 , -RXE_MAX_MR_INDEX =0x00010000 , +RXE_MAX_MR_INDEX =0x00040000 ,RXE_MIN_MW_INDEX =0x00010001 , RXE_MAX_MW_INDEX = 0x00020000, --2.27.0