Skip to contentSkip to conversation

staging: lustre: ko2iblnd: Adapt to the removal of ib_get_dma_mr()

PATCH New Doug Oucharek 1 participant 1 message Feb 12, 2017 Staging

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 staging: lustre: ko2iblnd: Adapt to the removal of ib_get_dma_mr()

In Linux kernel 4.9-rc1, the function ib_get_dma_mr() was removed and a second parameter was added to ib_alloc_pd(). As this broke the building of the ko2iblnd module in staging, the Kconfig for LNet has marked ko2iblnd as broken and stopped building it.

This patch fixes this breakage by:

- Removing the BROKEN tag from lnet/Kconfig.
- Make it so the module parameter map_on_demand can no longer be zero (we have to configure FMR/FastReg pools; it can no longer be off).
- No longer try to use the global DMA memory region, but make use of the FMR/FastReg pool for all RDMA Tx operations.
- Everywhere we are using the device DMA mr to derive the L-key for non-registered memory regions, use the pd->local_dma_lkey value instead.
- Make the default map_on_demand = 256. This will allow nodes with this patch to still connected to older nodes without this patch and FMR/FastReg turned off. When FMR/FastReg is turned off, we use 256 as the max frags so the two sides will still be able to communicate and work.

Intel-bug-id: https://jira.hpdd.intel.com/browse/LU-9026

---
 drivers/staging/lustre/lnet/Kconfig                |  1 -
 .../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c    | 69 +---------------------
 .../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h    |  3 -
 .../staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c | 17 +-----
 .../lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c  | 12 ++--
 5 files changed, 12 insertions(+), 90 deletions(-)
drivers/staging/lustre/lnet/Kconfig +0 -1
index 13b4327..2b59301 100644
@@ -35,7 +35,6 @@ config LNET_SELFTEST
 config LNET_XPRT_IB
 	tristate "LNET infiniband support"
 	depends on LNET && INFINIBAND && INFINIBAND_ADDR_TRANS
-	depends on BROKEN
 	default LNET && INFINIBAND
 	help
 	  This option allows the LNET users to use infiniband as an
drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c +2 -67

114 lines not on this page · raw message

drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h +0 -3
index 2cb4298..366372d 100644
@@ -172,7 +172,6 @@ struct kib_hca_dev {
 	__u64              ibh_page_mask;       /* page mask of current HCA */
 	int                ibh_mr_shift;        /* bits shift of max MR size */
 	__u64              ibh_mr_size;         /* size of MR */
-	struct ib_mr	   *ibh_mrs;		/* global MR */
 	struct ib_pd       *ibh_pd;             /* PD */
 	struct kib_dev	   *ibh_dev;		/* owner */
 	atomic_t           ibh_ref;             /* refcount */
@@ -978,8 +977,6 @@ static inline unsigned int kiblnd_sg_dma_len(struct ib_device *dev,
 #define KIBLND_CONN_PARAM(e)     ((e)->param.conn.private_data)
 #define KIBLND_CONN_PARAM_LEN(e) ((e)->param.conn.private_data_len)
 
-struct ib_mr *kiblnd_find_rd_dma_mr(struct lnet_ni *ni, struct kib_rdma_desc *rd,
-				    int negotiated_nfrags);
 void kiblnd_map_rx_descs(struct kib_conn *conn);
 void kiblnd_unmap_rx_descs(struct kib_conn *conn);
 void kiblnd_pool_free_node(struct kib_pool *pool, struct list_head *node);
drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c +2 -15
index e2f3f72..6eaa548 100644
@@ -157,7 +157,6 @@ static int kiblnd_init_rdma(struct kib_conn *conn, struct kib_tx *tx, int type,
 	struct kib_conn *conn = rx->rx_conn;
 	struct kib_net *net = conn->ibc_peer->ibp_ni->ni_data;
 	struct ib_recv_wr *bad_wrq = NULL;
-	struct ib_mr *mr = conn->ibc_hdev->ibh_mrs;
 	int rc;
 
 	LASSERT(net);
@@ -165,9 +164,8 @@ static int kiblnd_init_rdma(struct kib_conn *conn, struct kib_tx *tx, int type,
 	LASSERT(credit == IBLND_POSTRX_NO_CREDIT ||
 		credit == IBLND_POSTRX_PEER_CREDIT ||
 		credit == IBLND_POSTRX_RSRVD_CREDIT);
-	LASSERT(mr);
 
-	rx->rx_sge.lkey   = mr->lkey;
+	rx->rx_sge.lkey   = conn->ibc_hdev->ibh_pd->local_dma_lkey;
 	rx->rx_sge.addr   = rx->rx_msgaddr;
 	rx->rx_sge.length = IBLND_MSG_SIZE;
 
@@ -613,7 +611,6 @@ static int kiblnd_map_tx(lnet_ni_t *ni, struct kib_tx *tx, struct kib_rdma_desc
 {
 	struct kib_net *net = ni->ni_data;
 	struct kib_hca_dev *hdev = net->ibn_dev->ibd_hdev;
-	struct ib_mr *mr    = NULL;
 	__u32 nob;
 	int i;
 
@@ -635,14 +632,6 @@ static int kiblnd_map_tx(lnet_ni_t *ni, struct kib_tx *tx, struct kib_rdma_desc
 		nob += rd->rd_frags[i].rf_nob;
 	}
 
-	mr = kiblnd_find_rd_dma_mr(ni, rd, tx->tx_conn ?
-				   tx->tx_conn->ibc_max_frags : -1);
-	if (mr) {
-		/* found pre-mapping MR */
-		rd->rd_key = (rd != tx->tx_rd) ? mr->rkey : mr->lkey;
-		return 0;
-	}
-
 	if (net->ibn_fmr_ps)
 		return kiblnd_fmr_map_tx(net, tx, rd, nob);
 
@@ -1028,16 +1017,14 @@ static int kiblnd_map_tx(lnet_ni_t *ni, struct kib_tx *tx, struct kib_rdma_desc
 	struct ib_sge *sge = &tx->tx_sge[tx->tx_nwrq];
 	struct ib_rdma_wr *wrq = &tx->tx_wrq[tx->tx_nwrq];
 	int nob = offsetof(struct kib_msg, ibm_u) + body_nob;
-	struct ib_mr *mr = hdev->ibh_mrs;
 
 	LASSERT(tx->tx_nwrq >= 0);
 	LASSERT(tx->tx_nwrq < IBLND_MAX_RDMA_FRAGS + 1);
 	LASSERT(nob <= IBLND_MSG_SIZE);
-	LASSERT(mr);
 
 	kiblnd_init_msg(tx->tx_msg, type, body_nob);
 
-	sge->lkey   = mr->lkey;
+	sge->lkey   = hdev->ibh_pd->local_dma_lkey;
 	sge->addr   = tx->tx_msgaddr;
 	sge->length = nob;
 
drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c +8 -4
index 44e960f..3c81b527 100644
@@ -106,7 +106,8 @@
 module_param(concurrent_sends, int, 0444);
 MODULE_PARM_DESC(concurrent_sends, "send work-queue sizing");
 
-static int map_on_demand;
+#define IBLND_DEFAULT_MAP_ON_DEMAND IBLND_MAX_RDMA_FRAGS
+static int map_on_demand = IBLND_DEFAULT_MAP_ON_DEMAND;
 module_param(map_on_demand, int, 0444);
 MODULE_PARM_DESC(map_on_demand, "map on demand");
 
@@ -228,10 +229,13 @@ int kiblnd_tunables_setup(struct lnet_ni *ni)
 	if (tunables->lnd_peercredits_hiw >= ni->ni_peertxcredits)
 		tunables->lnd_peercredits_hiw = ni->ni_peertxcredits - 1;
 
-	if (tunables->lnd_map_on_demand < 0 ||
+	if (tunables->lnd_map_on_demand <= 0 ||
 	    tunables->lnd_map_on_demand > IBLND_MAX_RDMA_FRAGS) {
-		/* disable map-on-demand */
-		tunables->lnd_map_on_demand = 0;
+		/* Use the default */
+		CWARN("Invalid map_on_demand (%d), expects 1 - %d. Using default of %d\n",
+		      tunables->lnd_map_on_demand,
+		      IBLND_MAX_RDMA_FRAGS, IBLND_DEFAULT_MAP_ON_DEMAND);
+		tunables->lnd_map_on_demand = IBLND_DEFAULT_MAP_ON_DEMAND;
 	}
 
 	if (tunables->lnd_map_on_demand == 1) {
Signature · 1 line
1.8.3.1