Skip to contentSkip to conversation

RDMA/rxe: Bump up default maximum values used via uverbs

PATCH Superseded Rao Shoaib 3 participants 4 messages Jul 13, 2021

Subsystems (2)

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Rao Shoaib

author
#0 lore raw

PATCH 1/1 RDMA/rxe: Bump up default maximum values used via uverbs

From: Rao Shoaib <[email protected]>

In our internal testing we have found that the current maximum are too smalls. Ideally there should be no limits but currently maximum values are reported via ibv_query_device, so we have to keep maximum values but they have been made suffiently large.

---
 drivers/infiniband/sw/rxe/rxe_param.h | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)
drivers/infiniband/sw/rxe/rxe_param.h +14 -12
index 742e6ec93686..66a948adb1e1 100644
@@ -9,6 +9,8 @@
 
 #include <uapi/rdma/rdma_user_rxe.h>
 
+#define DEFAULT_MAX_VALUE (1 << 20)
+
 static inline enum ib_mtu rxe_mtu_int_to_enum(int mtu)
 {
 	if (mtu < 256)
@@ -37,7 +39,7 @@ static inline enum ib_mtu eth_mtu_int_to_enum(int mtu)
 enum rxe_device_param {
 	RXE_MAX_MR_SIZE			= -1ull,
 	RXE_PAGE_SIZE_CAP		= 0xfffff000,
-	RXE_MAX_QP_WR			= 0x4000,
+	RXE_MAX_QP_WR			= DEFAULT_MAX_VALUE,
 	RXE_DEVICE_CAP_FLAGS		= IB_DEVICE_BAD_PKEY_CNTR
 					| IB_DEVICE_BAD_QKEY_CNTR
 					| IB_DEVICE_AUTO_PATH_MIG
@@ -58,40 +60,40 @@ enum rxe_device_param {
 	RXE_MAX_INLINE_DATA		= RXE_MAX_WQE_SIZE -
 					  sizeof(struct rxe_send_wqe),
 	RXE_MAX_SGE_RD			= 32,
-	RXE_MAX_CQ			= 16384,
+	RXE_MAX_CQ			= DEFAULT_MAX_VALUE,
 	RXE_MAX_LOG_CQE			= 15,
-	RXE_MAX_PD			= 0x7ffc,
+	RXE_MAX_PD			= DEFAULT_MAX_VALUE,
 	RXE_MAX_QP_RD_ATOM		= 128,
 	RXE_MAX_RES_RD_ATOM		= 0x3f000,
 	RXE_MAX_QP_INIT_RD_ATOM		= 128,
 	RXE_MAX_MCAST_GRP		= 8192,
 	RXE_MAX_MCAST_QP_ATTACH		= 56,
 	RXE_MAX_TOT_MCAST_QP_ATTACH	= 0x70000,
-	RXE_MAX_AH			= 100,
-	RXE_MAX_SRQ_WR			= 0x4000,
+	RXE_MAX_AH			= DEFAULT_MAX_VALUE,
+	RXE_MAX_SRQ_WR			= DEFAULT_MAX_VALUE,
 	RXE_MIN_SRQ_WR			= 1,
 	RXE_MAX_SRQ_SGE			= 27,
 	RXE_MIN_SRQ_SGE			= 1,
 	RXE_MAX_FMR_PAGE_LIST_LEN	= 512,
-	RXE_MAX_PKEYS			= 1,
+	RXE_MAX_PKEYS			= DEFAULT_MAX_VALUE,
 	RXE_LOCAL_CA_ACK_DELAY		= 15,
 
-	RXE_MAX_UCONTEXT		= 512,
+	RXE_MAX_UCONTEXT		= DEFAULT_MAX_VALUE,
 
 	RXE_NUM_PORT			= 1,
 
-	RXE_MAX_QP			= 0x10000,
+	RXE_MAX_QP			= DEFAULT_MAX_VALUE,
 	RXE_MIN_QP_INDEX		= 16,
-	RXE_MAX_QP_INDEX		= 0x00020000,
+	RXE_MAX_QP_INDEX		= 0x00040000,
 
-	RXE_MAX_SRQ			= 0x00001000,
+	RXE_MAX_SRQ			= DEFAULT_MAX_VALUE,
 	RXE_MIN_SRQ_INDEX		= 0x00020001,
 	RXE_MAX_SRQ_INDEX		= 0x00040000,
 
-	RXE_MAX_MR			= 0x00001000,
+	RXE_MAX_MR			= DEFAULT_MAX_VALUE,
 	RXE_MAX_MW			= 0x00001000,
 	RXE_MIN_MR_INDEX		= 0x00000001,
-	RXE_MAX_MR_INDEX		= 0x00010000,
+	RXE_MAX_MR_INDEX		= 0x00040000,
 	RXE_MIN_MW_INDEX		= 0x00010001,
 	RXE_MAX_MW_INDEX		= 0x00020000,
 
Signature · 1 line
2.27.0
maintainer
#1 lore raw
Quoted 19 lines · Rao Shoaib

From: Rao Shoaib <[email protected]>

In our internal testing we have found that the current maximum are too smalls. Ideally there should be no limits but currently maximum values are reported via ibv_query_device, so we have to keep maximum values but they have been made suffiently large.

---
 drivers/infiniband/sw/rxe/rxe_param.h | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)
drivers/infiniband/sw/rxe/rxe_param.h +0 -0
index 742e6ec93686..66a948adb1e1 100644

drivers/infiniband/sw/rxe/rxe_param.h line 12

Quoted 4 lines
drivers/infiniband/sw/rxe/rxe_param.h +1 -0
@@ -9,6 +9,8 @@
 
 #include <uapi/rdma/rdma_user_rxe.h>
 
+#define DEFAULT_MAX_VALUE (1 << 20)

How do you get this DEFAULT_MAX_VALUE? From spec? or others?

Zhu Yanjun

Quoted 68 lines
drivers/infiniband/sw/rxe/rxe_param.h +13 -13
+
 static inline enum ib_mtu rxe_mtu_int_to_enum(int mtu)
 {
        if (mtu < 256)
@@ -37,7 +39,7 @@ static inline enum ib_mtu eth_mtu_int_to_enum(int mtu)
 enum rxe_device_param {
        RXE_MAX_MR_SIZE                 = -1ull,
        RXE_PAGE_SIZE_CAP               = 0xfffff000,
-       RXE_MAX_QP_WR                   = 0x4000,
+       RXE_MAX_QP_WR                   = DEFAULT_MAX_VALUE,
        RXE_DEVICE_CAP_FLAGS            = IB_DEVICE_BAD_PKEY_CNTR
                                        | IB_DEVICE_BAD_QKEY_CNTR
                                        | IB_DEVICE_AUTO_PATH_MIG
@@ -58,40 +60,40 @@ enum rxe_device_param {
        RXE_MAX_INLINE_DATA             = RXE_MAX_WQE_SIZE -
                                          sizeof(struct rxe_send_wqe),
        RXE_MAX_SGE_RD                  = 32,
-       RXE_MAX_CQ                      = 16384,
+       RXE_MAX_CQ                      = DEFAULT_MAX_VALUE,
        RXE_MAX_LOG_CQE                 = 15,
-       RXE_MAX_PD                      = 0x7ffc,
+       RXE_MAX_PD                      = DEFAULT_MAX_VALUE,
        RXE_MAX_QP_RD_ATOM              = 128,
        RXE_MAX_RES_RD_ATOM             = 0x3f000,
        RXE_MAX_QP_INIT_RD_ATOM         = 128,
        RXE_MAX_MCAST_GRP               = 8192,
        RXE_MAX_MCAST_QP_ATTACH         = 56,
        RXE_MAX_TOT_MCAST_QP_ATTACH     = 0x70000,
-       RXE_MAX_AH                      = 100,
-       RXE_MAX_SRQ_WR                  = 0x4000,
+       RXE_MAX_AH                      = DEFAULT_MAX_VALUE,
+       RXE_MAX_SRQ_WR                  = DEFAULT_MAX_VALUE,
        RXE_MIN_SRQ_WR                  = 1,
        RXE_MAX_SRQ_SGE                 = 27,
        RXE_MIN_SRQ_SGE                 = 1,
        RXE_MAX_FMR_PAGE_LIST_LEN       = 512,
-       RXE_MAX_PKEYS                   = 1,
+       RXE_MAX_PKEYS                   = DEFAULT_MAX_VALUE,
        RXE_LOCAL_CA_ACK_DELAY          = 15,
 
-       RXE_MAX_UCONTEXT                = 512,
+       RXE_MAX_UCONTEXT                = DEFAULT_MAX_VALUE,
 
        RXE_NUM_PORT                    = 1,
 
-       RXE_MAX_QP                      = 0x10000,
+       RXE_MAX_QP                      = DEFAULT_MAX_VALUE,
        RXE_MIN_QP_INDEX                = 16,
-       RXE_MAX_QP_INDEX                = 0x00020000,
+       RXE_MAX_QP_INDEX                = 0x00040000,
 
-       RXE_MAX_SRQ                     = 0x00001000,
+       RXE_MAX_SRQ                     = DEFAULT_MAX_VALUE,
        RXE_MIN_SRQ_INDEX               = 0x00020001,
        RXE_MAX_SRQ_INDEX               = 0x00040000,
 
-       RXE_MAX_MR                      = 0x00001000,
+       RXE_MAX_MR                      = DEFAULT_MAX_VALUE,
        RXE_MAX_MW                      = 0x00001000,
        RXE_MIN_MR_INDEX                = 0x00000001,
-       RXE_MAX_MR_INDEX                = 0x00010000,
+       RXE_MAX_MR_INDEX                = 0x00040000,
        RXE_MIN_MW_INDEX                = 0x00010001,
        RXE_MAX_MW_INDEX                = 0x00020000,
 
--

2.27.0

author
#2 lore raw
Quoted 20 lines · Zhu Yanjun
Quoted 19 lines · Rao Shoaib

From: Rao Shoaib <[email protected]>

In our internal testing we have found that the current maximum are too smalls. Ideally there should be no limits but currently maximum values are reported via ibv_query_device, so we have to keep maximum values but they have been made suffiently large.

---
  drivers/infiniband/sw/rxe/rxe_param.h | 26 ++++++++++++++------------
  1 file changed, 14 insertions(+), 12 deletions(-)
drivers/infiniband/sw/rxe/rxe_param.h +0 -0
index 742e6ec93686..66a948adb1e1 100644
@@ -9,6 +9,8 @@
 
Quoted 6 lines

drivers/infiniband/sw/rxe/rxe_param.h line 12

Quoted 3 lines
drivers/infiniband/sw/rxe/rxe_param.h +1 -0
  #include <uapi/rdma/rdma_user_rxe.h>
 
+#define DEFAULT_MAX_VALUE (1 << 20)

How do you get this DEFAULT_MAX_VALUE? From spec? or others?

Zhu Yanjun

The spec does not specify any limits. The value was proposed as part of discussion on my previous patch.

Shoaib

Quoted 69 lines
Quoted 69 lines
drivers/infiniband/sw/rxe/rxe_param.h +13 -13
+
  static inline enum ib_mtu rxe_mtu_int_to_enum(int mtu)
  {
         if (mtu < 256)
@@ -37,7 +39,7 @@ static inline enum ib_mtu eth_mtu_int_to_enum(int mtu)
  enum rxe_device_param {
         RXE_MAX_MR_SIZE                 = -1ull,
         RXE_PAGE_SIZE_CAP               = 0xfffff000,
-       RXE_MAX_QP_WR                   = 0x4000,
+       RXE_MAX_QP_WR                   = DEFAULT_MAX_VALUE,
         RXE_DEVICE_CAP_FLAGS            = IB_DEVICE_BAD_PKEY_CNTR
                                         | IB_DEVICE_BAD_QKEY_CNTR
                                         | IB_DEVICE_AUTO_PATH_MIG
@@ -58,40 +60,40 @@ enum rxe_device_param {
         RXE_MAX_INLINE_DATA             = RXE_MAX_WQE_SIZE -
                                           sizeof(struct rxe_send_wqe),
         RXE_MAX_SGE_RD                  = 32,
-       RXE_MAX_CQ                      = 16384,
+       RXE_MAX_CQ                      = DEFAULT_MAX_VALUE,
         RXE_MAX_LOG_CQE                 = 15,
-       RXE_MAX_PD                      = 0x7ffc,
+       RXE_MAX_PD                      = DEFAULT_MAX_VALUE,
         RXE_MAX_QP_RD_ATOM              = 128,
         RXE_MAX_RES_RD_ATOM             = 0x3f000,
         RXE_MAX_QP_INIT_RD_ATOM         = 128,
         RXE_MAX_MCAST_GRP               = 8192,
         RXE_MAX_MCAST_QP_ATTACH         = 56,
         RXE_MAX_TOT_MCAST_QP_ATTACH     = 0x70000,
-       RXE_MAX_AH                      = 100,
-       RXE_MAX_SRQ_WR                  = 0x4000,
+       RXE_MAX_AH                      = DEFAULT_MAX_VALUE,
+       RXE_MAX_SRQ_WR                  = DEFAULT_MAX_VALUE,
         RXE_MIN_SRQ_WR                  = 1,
         RXE_MAX_SRQ_SGE                 = 27,
         RXE_MIN_SRQ_SGE                 = 1,
         RXE_MAX_FMR_PAGE_LIST_LEN       = 512,
-       RXE_MAX_PKEYS                   = 1,
+       RXE_MAX_PKEYS                   = DEFAULT_MAX_VALUE,
         RXE_LOCAL_CA_ACK_DELAY          = 15,
 
-       RXE_MAX_UCONTEXT                = 512,
+       RXE_MAX_UCONTEXT                = DEFAULT_MAX_VALUE,
 
         RXE_NUM_PORT                    = 1,
 
-       RXE_MAX_QP                      = 0x10000,
+       RXE_MAX_QP                      = DEFAULT_MAX_VALUE,
         RXE_MIN_QP_INDEX                = 16,
-       RXE_MAX_QP_INDEX                = 0x00020000,
+       RXE_MAX_QP_INDEX                = 0x00040000,
 
-       RXE_MAX_SRQ                     = 0x00001000,
+       RXE_MAX_SRQ                     = DEFAULT_MAX_VALUE,
         RXE_MIN_SRQ_INDEX               = 0x00020001,
         RXE_MAX_SRQ_INDEX               = 0x00040000,
 
-       RXE_MAX_MR                      = 0x00001000,
+       RXE_MAX_MR                      = DEFAULT_MAX_VALUE,
         RXE_MAX_MW                      = 0x00001000,
         RXE_MIN_MR_INDEX                = 0x00000001,
-       RXE_MAX_MR_INDEX                = 0x00010000,
+       RXE_MAX_MR_INDEX                = 0x00040000,
         RXE_MIN_MW_INDEX                = 0x00010001,
         RXE_MAX_MW_INDEX                = 0x00020000,
 
--

2.27.0

#3 lore raw
In reply to Rao Shoaib
Thank you for the patch! Perhaps something to improve:

Hi Rao,

Thank you for the patch! Perhaps something to improve:

[auto build test WARNING on rdma/for-next]
[also build test WARNING on v5.14-rc1 next-20210713]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Rao-Shoaib/RDMA-rxe-Bump-up-default-maximum-values-used-via-uverbs/20210713-163821
base:   https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git for-next
config: x86_64-allyesconfig (attached as .config)
compiler: gcc-9 (Debian 9.3.0-22) 9.3.0
reproduce (this is a W=1 build):
        # https://github.com/0day-ci/linux/commit/823f0c9f1aff85ad2abeed713bf89fad22254f24
        git remote add linux-review https://github.com/0day-ci/linux
        git fetch --no-tags linux-review Rao-Shoaib/RDMA-rxe-Bump-up-default-maximum-values-used-via-uverbs/20210713-163821
        git checkout 823f0c9f1aff85ad2abeed713bf89fad22254f24
        # save the attached .config to linux build tree
        make W=1 ARCH=x86_64 

If you fix the issue, kindly add following tag as appropriate

All warnings (new ones prefixed by >>):

drivers/infiniband/sw/rxe/rxe.c: In function 'rxe_init_device_param':

>> drivers/infiniband/sw/rxe/rxe.c:71:26: warning: unsigned conversion from 'int' to 'u16' {aka 'short unsigned int'} changes value from '1048576' to '0' [-Woverflow]
      71 |  rxe->attr.max_pkeys   = RXE_MAX_PKEYS;
         |                          ^~~~~~~~~~~~~

Kconfig warnings: (for reference only)
WARNING: unmet direct dependencies detected for PHY_SPARX5_SERDES
Depends on (ARCH_SPARX5 || COMPILE_TEST && OF && HAS_IOMEM
Selected by
- SPARX5_SWITCH && NETDEVICES && ETHERNET && NET_VENDOR_MICROCHIP && NET_SWITCHDEV && HAS_IOMEM

vim +71 drivers/infiniband/sw/rxe/rxe.c

8700e3e7c4857d Moni Shoua        2016-06-16  39  
8700e3e7c4857d Moni Shoua        2016-06-16  40  /* initialize rxe device parameters */
befd8d98f230d9 Zhu Yanjun        2018-03-07  41  static void rxe_init_device_param(struct rxe_dev *rxe)
8700e3e7c4857d Moni Shoua        2016-06-16  42  {
8700e3e7c4857d Moni Shoua        2016-06-16  43  	rxe->max_inline_data			= RXE_MAX_INLINE_DATA;
8700e3e7c4857d Moni Shoua        2016-06-16  44  
0184afd15a141d Zhu Yanjun        2020-04-06  45  	rxe->attr.vendor_id			= RXE_VENDOR_ID;
8700e3e7c4857d Moni Shoua        2016-06-16  46  	rxe->attr.max_mr_size			= RXE_MAX_MR_SIZE;
8700e3e7c4857d Moni Shoua        2016-06-16  47  	rxe->attr.page_size_cap			= RXE_PAGE_SIZE_CAP;
8700e3e7c4857d Moni Shoua        2016-06-16  48  	rxe->attr.max_qp			= RXE_MAX_QP;
8700e3e7c4857d Moni Shoua        2016-06-16  49  	rxe->attr.max_qp_wr			= RXE_MAX_QP_WR;
8700e3e7c4857d Moni Shoua        2016-06-16  50  	rxe->attr.device_cap_flags		= RXE_DEVICE_CAP_FLAGS;
33023fb85a42b5 Steve Wise        2018-06-18  51  	rxe->attr.max_send_sge			= RXE_MAX_SGE;
33023fb85a42b5 Steve Wise        2018-06-18  52  	rxe->attr.max_recv_sge			= RXE_MAX_SGE;
8700e3e7c4857d Moni Shoua        2016-06-16  53  	rxe->attr.max_sge_rd			= RXE_MAX_SGE_RD;
8700e3e7c4857d Moni Shoua        2016-06-16  54  	rxe->attr.max_cq			= RXE_MAX_CQ;
8700e3e7c4857d Moni Shoua        2016-06-16  55  	rxe->attr.max_cqe			= (1 << RXE_MAX_LOG_CQE) - 1;
8700e3e7c4857d Moni Shoua        2016-06-16  56  	rxe->attr.max_mr			= RXE_MAX_MR;
af732adfacb2c6 Bob Pearson       2021-06-07  57  	rxe->attr.max_mw			= RXE_MAX_MW;
8700e3e7c4857d Moni Shoua        2016-06-16  58  	rxe->attr.max_pd			= RXE_MAX_PD;
8700e3e7c4857d Moni Shoua        2016-06-16  59  	rxe->attr.max_qp_rd_atom		= RXE_MAX_QP_RD_ATOM;
8700e3e7c4857d Moni Shoua        2016-06-16  60  	rxe->attr.max_res_rd_atom		= RXE_MAX_RES_RD_ATOM;
8700e3e7c4857d Moni Shoua        2016-06-16  61  	rxe->attr.max_qp_init_rd_atom		= RXE_MAX_QP_INIT_RD_ATOM;
0797e6f1a85872 Nathan Chancellor 2018-09-26  62  	rxe->attr.atomic_cap			= IB_ATOMIC_HCA;
8700e3e7c4857d Moni Shoua        2016-06-16  63  	rxe->attr.max_mcast_grp			= RXE_MAX_MCAST_GRP;
8700e3e7c4857d Moni Shoua        2016-06-16  64  	rxe->attr.max_mcast_qp_attach		= RXE_MAX_MCAST_QP_ATTACH;
8700e3e7c4857d Moni Shoua        2016-06-16  65  	rxe->attr.max_total_mcast_qp_attach	= RXE_MAX_TOT_MCAST_QP_ATTACH;
8700e3e7c4857d Moni Shoua        2016-06-16  66  	rxe->attr.max_ah			= RXE_MAX_AH;
8700e3e7c4857d Moni Shoua        2016-06-16  67  	rxe->attr.max_srq			= RXE_MAX_SRQ;
8700e3e7c4857d Moni Shoua        2016-06-16  68  	rxe->attr.max_srq_wr			= RXE_MAX_SRQ_WR;
8700e3e7c4857d Moni Shoua        2016-06-16  69  	rxe->attr.max_srq_sge			= RXE_MAX_SRQ_SGE;
8700e3e7c4857d Moni Shoua        2016-06-16  70  	rxe->attr.max_fast_reg_page_list_len	= RXE_MAX_FMR_PAGE_LIST_LEN;
8700e3e7c4857d Moni Shoua        2016-06-16 @71  	rxe->attr.max_pkeys			= RXE_MAX_PKEYS;
8700e3e7c4857d Moni Shoua        2016-06-16  72  	rxe->attr.local_ca_ack_delay		= RXE_LOCAL_CA_ACK_DELAY;
d0ca2c35dd15a3 Zhu Yanjun        2020-03-23  73  	addrconf_addr_eui48((unsigned char *)&rxe->attr.sys_image_guid,
d0ca2c35dd15a3 Zhu Yanjun        2020-03-23  74  			rxe->ndev->dev_addr);
8700e3e7c4857d Moni Shoua        2016-06-16  75  
8700e3e7c4857d Moni Shoua        2016-06-16  76  	rxe->max_ucontext			= RXE_MAX_UCONTEXT;
8700e3e7c4857d Moni Shoua        2016-06-16  77  }
8700e3e7c4857d Moni Shoua        2016-06-16  78  

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org