Skip to contentSkip to conversation

rdma_rxe: API extensions

PATCH In review Bob Pearson 2 participants 22 messages Oct 1, 2020 – Oct 5, 2020

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

v7 cover rdma_rxe: API extensions

V7:
Ported this patch series forward to the current state of for-next.

MW now being allocated in core.

Split up rxe_pool changes into patches as requested by Jason. Added some better comments.
Added error returns via ERR_PTR for APIs that return pointers.

Added some additional validation for rkeys in rxe_resp based on chapter 9 of IBA.

Minor cleanup to rxe_mr which had duplicate entries.

V6:
Fixed two issues raised by Jason and Zhu.

Undid the replacement of rwlocks by spinlocks in patch 10/12. On further reading it turns out rwlocks were the better choice.

Missing prototype for rxe_invalidate_mr. This was caused by a regression in patch 05/12 which had dropped the actual use of the routine as well as the prototype. Fixed.

V5:
This patch series is a collection of API extensions for the rdma_rxe driver. With this patch set installed there are no errors in pyverbs run-tests and 31 tests are skipped down from 56. The remaining skipped test cases include
- XRC tests
- ODP tests
- Parent device tests
- Import tests
- Device memory
- MLX5 specific tests
- EFA tests

It continues from the previous (v4) set which implemented memory windows and has had a number of individual patches picked up in for-next.

This set (v5) includes:
Ported to current head of tree
Memory windows patches not yet picked up kernel support for the extended user space APIs:
- ibv_query_device_ex
- ibv_create_cq_ex
- ibv_create_qp_ex
Fixes for multicast which is not currently working

This patch set depends on a matching rdma-core user space library patch set.

In order to run correctly it is necessary to configure by hand the EUI64 link local IPV6 address on systems which use a random link local address (like Ubuntu).

Bob Pearson (19):
rdma_rxe: Separat MEM into MR and MW objects.
rdma_rxe: Enable MW objects
rdma_rxe: Let pools support both keys and indices
rdma_rxe: make pool return values position independent
rdma_rxe: remove void * parameters in pool APIs
rdma_rxe: add alloc_mw and dealloc_mw verbs
rdma_rxe: Add bind_mw and invalidate_mw verbs
rdma_rxe: Add memory access through MWs
rdma_rxe: Add locked and unlocked pool APIs
rdma_rxe: Add support for ibv_query_device_ex
rdma_rxe: Add support for extended CQ operations
rdma_rxe: Add support for extended QP operations
rdma_rxe: Fix mcast group allocation bug
rdma_rxe: Fix bugs in the multicast receive path
rdma_rxe: handle ERR_PTR returns from pool
rdma_rxe: remove duplicate entries in struct rxe_mr
rdma_rme: removed unused RXE_MR_TYPE_FMR
rdma_rxe: add rkey validation checks for MR and MW
rdma_rxe: moved rxe_xmit_packet to rxe_net.c

 drivers/infiniband/sw/rxe/Makefile     |   1 +
 drivers/infiniband/sw/rxe/rxe.c        | 100 ++--
 drivers/infiniband/sw/rxe/rxe_comp.c   |  12 +-
 drivers/infiniband/sw/rxe/rxe_loc.h    |  91 ++--
 drivers/infiniband/sw/rxe/rxe_mcast.c  | 118 +++--
 drivers/infiniband/sw/rxe/rxe_mr.c     | 396 ++++++++-------
 drivers/infiniband/sw/rxe/rxe_mw.c     | 420 ++++++++++++++++
 drivers/infiniband/sw/rxe/rxe_net.c    |  47 +-
 drivers/infiniband/sw/rxe/rxe_opcode.c |  11 +-
 drivers/infiniband/sw/rxe/rxe_opcode.h |   1 -
 drivers/infiniband/sw/rxe/rxe_param.h  |  10 +-
 drivers/infiniband/sw/rxe/rxe_pool.c   | 653 ++++++++++++++++++-------
 drivers/infiniband/sw/rxe/rxe_pool.h   | 148 +++---
 drivers/infiniband/sw/rxe/rxe_recv.c   |  67 ++-
 drivers/infiniband/sw/rxe/rxe_req.c    | 145 ++++--
 drivers/infiniband/sw/rxe/rxe_resp.c   | 188 +++++--
 drivers/infiniband/sw/rxe/rxe_verbs.c  | 118 +++--
 drivers/infiniband/sw/rxe/rxe_verbs.h  |  71 ++-
 include/uapi/rdma/rdma_user_rxe.h      |  68 ++-
 19 files changed, 1930 insertions(+), 735 deletions(-)
 create mode 100644 drivers/infiniband/sw/rxe/rxe_mw.c
Signature · 1 line
2.25.1
#1 lore raw

PATCH v7 1/19 rdma_rxe: Separat MEM into MR and MW objects.

In the original rxe implementation it was intended to use a common object to represent MRs and MWs but it became clear that they are different enough to separate these into two objects.

This allows replacing the mem name with mr for MRs which is more consistent with the style for the other objects and less likely to be confusing. This is a long patch that mostly changes mem to mr where it makes sense and adds a new rxe_mw struct.

---
 drivers/infiniband/sw/rxe/rxe_comp.c  |   4 +-
 drivers/infiniband/sw/rxe/rxe_loc.h   |  26 +--
 drivers/infiniband/sw/rxe/rxe_mr.c    | 264 +++++++++++++-------------
 drivers/infiniband/sw/rxe/rxe_pool.c  |   8 +-
 drivers/infiniband/sw/rxe/rxe_req.c   |   6 +-
 drivers/infiniband/sw/rxe/rxe_resp.c  |  30 +--
 drivers/infiniband/sw/rxe/rxe_verbs.c |  18 +-
 drivers/infiniband/sw/rxe/rxe_verbs.h |  51 ++---
 8 files changed, 204 insertions(+), 203 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +2 -2
index 0a1e6393250b..5dc86c9e74c2 100644
@@ -345,7 +345,7 @@ static inline enum comp_state do_read(struct rxe_qp *qp,
 
 	ret = copy_data(qp->pd, IB_ACCESS_LOCAL_WRITE,
 			&wqe->dma, payload_addr(pkt),
-			payload_size(pkt), to_mem_obj, NULL);
+			payload_size(pkt), to_mr_obj, NULL);
 	if (ret)
 		return COMPST_ERROR;
 
@@ -365,7 +365,7 @@ static inline enum comp_state do_atomic(struct rxe_qp *qp,
 
 	ret = copy_data(qp->pd, IB_ACCESS_LOCAL_WRITE,
 			&wqe->dma, &atomic_orig,
-			sizeof(u64), to_mem_obj, NULL);
+			sizeof(u64), to_mr_obj, NULL);
 	if (ret)
 		return COMPST_ERROR;
 	else
drivers/infiniband/sw/rxe/rxe_loc.h +13 -13
index 0d758760b9ae..9ec6bff6863f 100644
@@ -72,40 +72,40 @@ int rxe_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);
 
 /* rxe_mr.c */
 enum copy_direction {
-	to_mem_obj,
-	from_mem_obj,
+	to_mr_obj,
+	from_mr_obj,
 };
 
-void rxe_mem_init_dma(struct rxe_pd *pd,
-		      int access, struct rxe_mem *mem);
+void rxe_mr_init_dma(struct rxe_pd *pd,
+		      int access, struct rxe_mr *mr);
 
-int rxe_mem_init_user(struct rxe_pd *pd, u64 start,
+int rxe_mr_init_user(struct rxe_pd *pd, u64 start,
 		      u64 length, u64 iova, int access, struct ib_udata *udata,
-		      struct rxe_mem *mr);
+		      struct rxe_mr *mr);
 
-int rxe_mem_init_fast(struct rxe_pd *pd,
-		      int max_pages, struct rxe_mem *mem);
+int rxe_mr_init_fast(struct rxe_pd *pd,
+		      int max_pages, struct rxe_mr *mr);
 
-int rxe_mem_copy(struct rxe_mem *mem, u64 iova, void *addr,
+int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr,
 		 int length, enum copy_direction dir, u32 *crcp);
 
 int copy_data(struct rxe_pd *pd, int access,
 	      struct rxe_dma_info *dma, void *addr, int length,
 	      enum copy_direction dir, u32 *crcp);
 
-void *iova_to_vaddr(struct rxe_mem *mem, u64 iova, int length);
+void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length);
 
 enum lookup_type {
 	lookup_local,
 	lookup_remote,
 };
 
-struct rxe_mem *lookup_mem(struct rxe_pd *pd, int access, u32 key,
+struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
 			   enum lookup_type type);
 
-int mem_check_range(struct rxe_mem *mem, u64 iova, size_t length);
+int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);
 
-void rxe_mem_cleanup(struct rxe_pool_entry *arg);
+void rxe_mr_cleanup(struct rxe_pool_entry *arg);
 
 int advance_dma_data(struct rxe_dma_info *dma, unsigned int length);
 
drivers/infiniband/sw/rxe/rxe_mr.c +132 -132

518 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_pool.c +3 -5
index b374eb53e2fe..32ba47d143f3 100644
@@ -8,8 +8,6 @@
 #include "rxe_loc.h"
 
 /* info about object pools
- * note that mr and mw share a single index space
- * so that one can map an lkey to the correct type of object
  */
 struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
 	[RXE_TYPE_UC] = {
@@ -50,15 +48,15 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
 	},
 	[RXE_TYPE_MR] = {
 		.name		= "rxe-mr",
-		.size		= sizeof(struct rxe_mem),
-		.cleanup	= rxe_mem_cleanup,
+		.size		= sizeof(struct rxe_mr),
+		.cleanup	= rxe_mr_cleanup,
 		.flags		= RXE_POOL_INDEX,
 		.max_index	= RXE_MAX_MR_INDEX,
 		.min_index	= RXE_MIN_MR_INDEX,
 	},
 	[RXE_TYPE_MW] = {
 		.name		= "rxe-mw",
-		.size		= sizeof(struct rxe_mem),
+		.size		= sizeof(struct rxe_mw),
 		.flags		= RXE_POOL_INDEX,
 		.max_index	= RXE_MAX_MW_INDEX,
 		.min_index	= RXE_MIN_MW_INDEX,
drivers/infiniband/sw/rxe/rxe_req.c +3 -3
index e27585ce9eb7..57236d8c2146 100644
@@ -465,7 +465,7 @@ static int fill_packet(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
 		} else {
 			err = copy_data(qp->pd, 0, &wqe->dma,
 					payload_addr(pkt), paylen,
-					from_mem_obj,
+					from_mr_obj,
 					&crc);
 			if (err)
 				return err;
@@ -597,7 +597,7 @@ int rxe_requester(void *arg)
 	if (wqe->mask & WR_REG_MASK) {
 		if (wqe->wr.opcode == IB_WR_LOCAL_INV) {
 			struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
-			struct rxe_mem *rmr;
+			struct rxe_mr *rmr;
 
 			rmr = rxe_pool_get_index(&rxe->mr_pool,
 						 wqe->wr.ex.invalidate_rkey >> 8);
@@ -613,7 +613,7 @@ int rxe_requester(void *arg)
 			wqe->state = wqe_state_done;
 			wqe->status = IB_WC_SUCCESS;
 		} else if (wqe->wr.opcode == IB_WR_REG_MR) {
-			struct rxe_mem *rmr = to_rmr(wqe->wr.wr.reg.mr);
+			struct rxe_mr *rmr = to_rmr(wqe->wr.wr.reg.mr);
 
 			rmr->state = RXE_MEM_STATE_VALID;
 			rmr->access = wqe->wr.wr.reg.access;
drivers/infiniband/sw/rxe/rxe_resp.c +15 -15
index c7e3b6a4af38..69867bf39cfb 100644
@@ -390,7 +390,7 @@ static enum resp_states check_length(struct rxe_qp *qp,
 static enum resp_states check_rkey(struct rxe_qp *qp,
 				   struct rxe_pkt_info *pkt)
 {
-	struct rxe_mem *mem = NULL;
+	struct rxe_mr *mr = NULL;
 	u64 va;
 	u32 rkey;
 	u32 resid;
@@ -429,18 +429,18 @@ static enum resp_states check_rkey(struct rxe_qp *qp,
 	resid	= qp->resp.resid;
 	pktlen	= payload_size(pkt);
 
-	mem = lookup_mem(qp->pd, access, rkey, lookup_remote);
-	if (!mem) {
+	mr = lookup_mr(qp->pd, access, rkey, lookup_remote);
+	if (!mr) {
 		state = RESPST_ERR_RKEY_VIOLATION;
 		goto err;
 	}
 
-	if (unlikely(mem->state == RXE_MEM_STATE_FREE)) {
+	if (unlikely(mr->state == RXE_MEM_STATE_FREE)) {
 		state = RESPST_ERR_RKEY_VIOLATION;
 		goto err;
 	}
 
-	if (mem_check_range(mem, va, resid)) {
+	if (mr_check_range(mr, va, resid)) {
 		state = RESPST_ERR_RKEY_VIOLATION;
 		goto err;
 	}
@@ -468,12 +468,12 @@ static enum resp_states check_rkey(struct rxe_qp *qp,
 
 	WARN_ON_ONCE(qp->resp.mr);
 
-	qp->resp.mr = mem;
+	qp->resp.mr = mr;
 	return RESPST_EXECUTE;
 
 err:
-	if (mem)
-		rxe_drop_ref(mem);
+	if (mr)
+		rxe_drop_ref(mr);
 	return state;
 }
 
@@ -483,7 +483,7 @@ static enum resp_states send_data_in(struct rxe_qp *qp, void *data_addr,
 	int err;
 
 	err = copy_data(qp->pd, IB_ACCESS_LOCAL_WRITE, &qp->resp.wqe->dma,
-			data_addr, data_len, to_mem_obj, NULL);
+			data_addr, data_len, to_mr_obj, NULL);
 	if (unlikely(err))
 		return (err == -ENOSPC) ? RESPST_ERR_LENGTH
 					: RESPST_ERR_MALFORMED_WQE;
@@ -498,8 +498,8 @@ static enum resp_states write_data_in(struct rxe_qp *qp,
 	int	err;
 	int data_len = payload_size(pkt);
 
-	err = rxe_mem_copy(qp->resp.mr, qp->resp.va, payload_addr(pkt),
-			   data_len, to_mem_obj, NULL);
+	err = rxe_mr_copy(qp->resp.mr, qp->resp.va, payload_addr(pkt),
+			   data_len, to_mr_obj, NULL);
 	if (err) {
 		rc = RESPST_ERR_RKEY_VIOLATION;
 		goto out;
@@ -521,7 +521,7 @@ static enum resp_states process_atomic(struct rxe_qp *qp,
 	u64 iova = atmeth_va(pkt);
 	u64 *vaddr;
 	enum resp_states ret;
-	struct rxe_mem *mr = qp->resp.mr;
+	struct rxe_mr *mr = qp->resp.mr;
 
 	if (mr->state != RXE_MEM_STATE_VALID) {
 		ret = RESPST_ERR_RKEY_VIOLATION;
@@ -700,8 +700,8 @@ static enum resp_states read_reply(struct rxe_qp *qp,
 	if (!skb)
 		return RESPST_ERR_RNR;
 
-	err = rxe_mem_copy(res->read.mr, res->read.va, payload_addr(&ack_pkt),
-			   payload, from_mem_obj, &icrc);
+	err = rxe_mr_copy(res->read.mr, res->read.va, payload_addr(&ack_pkt),
+			   payload, from_mr_obj, &icrc);
 	if (err)
 		pr_err("Failed copying memory\n");
 
@@ -883,7 +883,7 @@ static enum resp_states do_complete(struct rxe_qp *qp,
 			}
 
 			if (pkt->mask & RXE_IETH_MASK) {
-				struct rxe_mem *rmr;
+				struct rxe_mr *rmr;
 
 				wc->wc_flags |= IB_WC_WITH_INVALIDATE;
 				wc->ex.invalidate_rkey = ieth_rkey(pkt);
drivers/infiniband/sw/rxe/rxe_verbs.c +9 -9
index f368dc16281a..664bcb025aae 100644
@@ -867,7 +867,7 @@ static struct ib_mr *rxe_get_dma_mr(struct ib_pd *ibpd, int access)
 {
 	struct rxe_dev *rxe = to_rdev(ibpd->device);
 	struct rxe_pd *pd = to_rpd(ibpd);
-	struct rxe_mem *mr;
+	struct rxe_mr *mr;
 
 	mr = rxe_alloc(&rxe->mr_pool);
 	if (!mr)
@@ -875,7 +875,7 @@ static struct ib_mr *rxe_get_dma_mr(struct ib_pd *ibpd, int access)
 
 	rxe_add_index(mr);
 	rxe_add_ref(pd);
-	rxe_mem_init_dma(pd, access, mr);
+	rxe_mr_init_dma(pd, access, mr);
 
 	return &mr->ibmr;
 }
@@ -889,7 +889,7 @@ static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd,
 	int err;
 	struct rxe_dev *rxe = to_rdev(ibpd->device);
 	struct rxe_pd *pd = to_rpd(ibpd);
-	struct rxe_mem *mr;
+	struct rxe_mr *mr;
 
 	mr = rxe_alloc(&rxe->mr_pool);
 	if (!mr) {
@@ -901,7 +901,7 @@ static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd,
 
 	rxe_add_ref(pd);
 
-	err = rxe_mem_init_user(pd, start, length, iova,
+	err = rxe_mr_init_user(pd, start, length, iova,
 				access, udata, mr);
 	if (err)
 		goto err3;
@@ -918,7 +918,7 @@ static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd,
 
 static int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
 {
-	struct rxe_mem *mr = to_rmr(ibmr);
+	struct rxe_mr *mr = to_rmr(ibmr);
 
 	mr->state = RXE_MEM_STATE_ZOMBIE;
 	rxe_drop_ref(mr->pd);
@@ -932,7 +932,7 @@ static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
 {
 	struct rxe_dev *rxe = to_rdev(ibpd->device);
 	struct rxe_pd *pd = to_rpd(ibpd);
-	struct rxe_mem *mr;
+	struct rxe_mr *mr;
 	int err;
 
 	if (mr_type != IB_MR_TYPE_MEM_REG)
@@ -948,7 +948,7 @@ static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
 
 	rxe_add_ref(pd);
 
-	err = rxe_mem_init_fast(pd, max_num_sg, mr);
+	err = rxe_mr_init_fast(pd, max_num_sg, mr);
 	if (err)
 		goto err2;
 
@@ -964,7 +964,7 @@ static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
 
 static int rxe_set_page(struct ib_mr *ibmr, u64 addr)
 {
-	struct rxe_mem *mr = to_rmr(ibmr);
+	struct rxe_mr *mr = to_rmr(ibmr);
 	struct rxe_map *map;
 	struct rxe_phys_buf *buf;
 
@@ -984,7 +984,7 @@ static int rxe_set_page(struct ib_mr *ibmr, u64 addr)
 static int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 			 int sg_nents, unsigned int *sg_offset)
 {
-	struct rxe_mem *mr = to_rmr(ibmr);
+	struct rxe_mr *mr = to_rmr(ibmr);
 	int n;
 
 	mr->nbuf = 0;
drivers/infiniband/sw/rxe/rxe_verbs.h +27 -24

106 lines not on this page · raw message

Signature · 1 line
2.25.1
#2 lore raw

PATCH v7 2/19 rdma_rxe: Enable MW objects

In reply to Bob Pearson

Change parameters in rxe_param.h so that MAX_MW is the same as MAX_MR. Set device attribute in rxe.c so max_mw = MAX_MW.

---
 drivers/infiniband/sw/rxe/rxe.c       |  1 +
 drivers/infiniband/sw/rxe/rxe_param.h | 10 ++++++----
 2 files changed, 7 insertions(+), 4 deletions(-)
drivers/infiniband/sw/rxe/rxe.c +1 -0
index 95f0de0c8b49..8e0f9c489cab 100644
@@ -54,6 +54,7 @@ static void rxe_init_device_param(struct rxe_dev *rxe)
 	rxe->attr.max_cq			= RXE_MAX_CQ;
 	rxe->attr.max_cqe			= (1 << RXE_MAX_LOG_CQE) - 1;
 	rxe->attr.max_mr			= RXE_MAX_MR;
+	rxe->attr.max_mw			= RXE_MAX_MW;
 	rxe->attr.max_pd			= RXE_MAX_PD;
 	rxe->attr.max_qp_rd_atom		= RXE_MAX_QP_RD_ATOM;
 	rxe->attr.max_res_rd_atom		= RXE_MAX_RES_RD_ATOM;
drivers/infiniband/sw/rxe/rxe_param.h +6 -4
index 25ab50d9b7c2..4ebb3da8c07d 100644
@@ -58,7 +58,8 @@ enum rxe_device_param {
 	RXE_MAX_SGE_RD			= 32,
 	RXE_MAX_CQ			= 16384,
 	RXE_MAX_LOG_CQE			= 15,
-	RXE_MAX_MR			= 256 * 1024,
+	RXE_MAX_MR			= 0x40000,
+	RXE_MAX_MW			= 0x40000,
 	RXE_MAX_PD			= 0x7ffc,
 	RXE_MAX_QP_RD_ATOM		= 128,
 	RXE_MAX_RES_RD_ATOM		= 0x3f000,
@@ -87,9 +88,10 @@ enum rxe_device_param {
 	RXE_MAX_SRQ_INDEX		= 0x00040000,
 
 	RXE_MIN_MR_INDEX		= 0x00000001,
-	RXE_MAX_MR_INDEX		= 0x00040000,
-	RXE_MIN_MW_INDEX		= 0x00040001,
-	RXE_MAX_MW_INDEX		= 0x00060000,
+	RXE_MAX_MR_INDEX		= RXE_MIN_MR_INDEX + RXE_MAX_MR - 1,
+	RXE_MIN_MW_INDEX		= RXE_MIN_MR_INDEX + RXE_MAX_MR,
+	RXE_MAX_MW_INDEX		= RXE_MIN_MW_INDEX + RXE_MAX_MW - 1,
+
 	RXE_MAX_PKT_PER_ACK		= 64,
 
 	RXE_MAX_UNACKED_PSNS		= 128,
Signature · 1 line
2.25.1
#3 lore raw

PATCH v7 3/19 rdma_rxe: Let pools support both keys and indices

In reply to Bob Pearson

Allow both indices and keys to exist for objects in pools. Previously you were limited to one or the other. This will support allowing the keys on MWs to change.

---
 drivers/infiniband/sw/rxe/rxe_pool.c | 73 ++++++++++++++--------------
 drivers/infiniband/sw/rxe/rxe_pool.h | 32 +++++++-----
 2 files changed, 58 insertions(+), 47 deletions(-)
drivers/infiniband/sw/rxe/rxe_pool.c +37 -36
index 32ba47d143f3..30b8f037ee20 100644
@@ -92,18 +92,18 @@ static int rxe_pool_init_index(struct rxe_pool *pool, u32 max, u32 min)
 		goto out;
 	}
 
-	pool->max_index = max;
-	pool->min_index = min;
+	pool->index.max_index = max;
+	pool->index.min_index = min;
 
 	size = BITS_TO_LONGS(max - min + 1) * sizeof(long);
-	pool->table = kmalloc(size, GFP_KERNEL);
-	if (!pool->table) {
+	pool->index.table = kmalloc(size, GFP_KERNEL);
+	if (!pool->index.table) {
 		err = -ENOMEM;
 		goto out;
 	}
 
-	pool->table_size = size;
-	bitmap_zero(pool->table, max - min + 1);
+	pool->index.table_size = size;
+	bitmap_zero(pool->index.table, max - min + 1);
 
 out:
 	return err;
@@ -125,7 +125,8 @@ int rxe_pool_init(
 	pool->max_elem		= max_elem;
 	pool->elem_size		= ALIGN(size, RXE_POOL_ALIGN);
 	pool->flags		= rxe_type_info[type].flags;
-	pool->tree		= RB_ROOT;
+	pool->index.tree	= RB_ROOT;
+	pool->key.tree		= RB_ROOT;
 	pool->cleanup		= rxe_type_info[type].cleanup;
 
 	atomic_set(&pool->num_elem, 0);
@@ -143,8 +144,8 @@ int rxe_pool_init(
 	}
 
 	if (rxe_type_info[type].flags & RXE_POOL_KEY) {
-		pool->key_offset = rxe_type_info[type].key_offset;
-		pool->key_size = rxe_type_info[type].key_size;
+		pool->key.key_offset = rxe_type_info[type].key_offset;
+		pool->key.key_size = rxe_type_info[type].key_size;
 	}
 
 	pool->state = RXE_POOL_STATE_VALID;
@@ -158,7 +159,7 @@ static void rxe_pool_release(struct kref *kref)
 	struct rxe_pool *pool = container_of(kref, struct rxe_pool, ref_cnt);
 
 	pool->state = RXE_POOL_STATE_INVALID;
-	kfree(pool->table);
+	kfree(pool->index.table);
 }
 
 static void rxe_pool_put(struct rxe_pool *pool)
@@ -183,27 +184,27 @@ void rxe_pool_cleanup(struct rxe_pool *pool)
 static u32 alloc_index(struct rxe_pool *pool)
 {
 	u32 index;
-	u32 range = pool->max_index - pool->min_index + 1;
+	u32 range = pool->index.max_index - pool->index.min_index + 1;
 
-	index = find_next_zero_bit(pool->table, range, pool->last);
+	index = find_next_zero_bit(pool->index.table, range, pool->index.last);
 	if (index >= range)
-		index = find_first_zero_bit(pool->table, range);
+		index = find_first_zero_bit(pool->index.table, range);
 
 	WARN_ON_ONCE(index >= range);
-	set_bit(index, pool->table);
-	pool->last = index;
-	return index + pool->min_index;
+	set_bit(index, pool->index.table);
+	pool->index.last = index;
+	return index + pool->index.min_index;
 }
 
 static void insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new)
 {
-	struct rb_node **link = &pool->tree.rb_node;
+	struct rb_node **link = &pool->index.tree.rb_node;
 	struct rb_node *parent = NULL;
 	struct rxe_pool_entry *elem;
 
 	while (*link) {
 		parent = *link;
-		elem = rb_entry(parent, struct rxe_pool_entry, node);
+		elem = rb_entry(parent, struct rxe_pool_entry, index_node);
 
 		if (elem->index == new->index) {
 			pr_warn("element already exists!\n");
@@ -216,25 +217,25 @@ static void insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new)
 			link = &(*link)->rb_right;
 	}
 
-	rb_link_node(&new->node, parent, link);
-	rb_insert_color(&new->node, &pool->tree);
+	rb_link_node(&new->index_node, parent, link);
+	rb_insert_color(&new->index_node, &pool->index.tree);
 out:
 	return;
 }
 
 static void insert_key(struct rxe_pool *pool, struct rxe_pool_entry *new)
 {
-	struct rb_node **link = &pool->tree.rb_node;
+	struct rb_node **link = &pool->key.tree.rb_node;
 	struct rb_node *parent = NULL;
 	struct rxe_pool_entry *elem;
 	int cmp;
 
 	while (*link) {
 		parent = *link;
-		elem = rb_entry(parent, struct rxe_pool_entry, node);
+		elem = rb_entry(parent, struct rxe_pool_entry, key_node);
 
-		cmp = memcmp((u8 *)elem + pool->key_offset,
-			     (u8 *)new + pool->key_offset, pool->key_size);
+		cmp = memcmp((u8 *)elem + pool->key.key_offset,
+			     (u8 *)new + pool->key.key_offset, pool->key.key_size);
 
 		if (cmp == 0) {
 			pr_warn("key already exists!\n");
@@ -247,8 +248,8 @@ static void insert_key(struct rxe_pool *pool, struct rxe_pool_entry *new)
 			link = &(*link)->rb_right;
 	}
 
-	rb_link_node(&new->node, parent, link);
-	rb_insert_color(&new->node, &pool->tree);
+	rb_link_node(&new->key_node, parent, link);
+	rb_insert_color(&new->key_node, &pool->key.tree);
 out:
 	return;
 }
@@ -260,7 +261,7 @@ void rxe_add_key(void *arg, void *key)
 	unsigned long flags;
 
 	write_lock_irqsave(&pool->pool_lock, flags);
-	memcpy((u8 *)elem + pool->key_offset, key, pool->key_size);
+	memcpy((u8 *)elem + pool->key.key_offset, key, pool->key.key_size);
 	insert_key(pool, elem);
 	write_unlock_irqrestore(&pool->pool_lock, flags);
 }
@@ -272,7 +273,7 @@ void rxe_drop_key(void *arg)
 	unsigned long flags;
 
 	write_lock_irqsave(&pool->pool_lock, flags);
-	rb_erase(&elem->node, &pool->tree);
+	rb_erase(&elem->key_node, &pool->key.tree);
 	write_unlock_irqrestore(&pool->pool_lock, flags);
 }
 
@@ -295,8 +296,8 @@ void rxe_drop_index(void *arg)
 	unsigned long flags;
 
 	write_lock_irqsave(&pool->pool_lock, flags);
-	clear_bit(elem->index - pool->min_index, pool->table);
-	rb_erase(&elem->node, &pool->tree);
+	clear_bit(elem->index - pool->index.min_index, pool->index.table);
+	rb_erase(&elem->index_node, &pool->index.tree);
 	write_unlock_irqrestore(&pool->pool_lock, flags);
 }
 
@@ -400,10 +401,10 @@ void *rxe_pool_get_index(struct rxe_pool *pool, u32 index)
 	if (pool->state != RXE_POOL_STATE_VALID)
 		goto out;
 
-	node = pool->tree.rb_node;
+	node = pool->index.tree.rb_node;
 
 	while (node) {
-		elem = rb_entry(node, struct rxe_pool_entry, node);
+		elem = rb_entry(node, struct rxe_pool_entry, index_node);
 
 		if (elem->index > index)
 			node = node->rb_left;
@@ -432,13 +433,13 @@ void *rxe_pool_get_key(struct rxe_pool *pool, void *key)
 	if (pool->state != RXE_POOL_STATE_VALID)
 		goto out;
 
-	node = pool->tree.rb_node;
+	node = pool->key.tree.rb_node;
 
 	while (node) {
-		elem = rb_entry(node, struct rxe_pool_entry, node);
+		elem = rb_entry(node, struct rxe_pool_entry, key_node);
 
-		cmp = memcmp((u8 *)elem + pool->key_offset,
-			     key, pool->key_size);
+		cmp = memcmp((u8 *)elem + pool->key.key_offset,
+			     key, pool->key.key_size);
 
 		if (cmp > 0)
 			node = node->rb_left;
drivers/infiniband/sw/rxe/rxe_pool.h +21 -11
index 432745ffc8d4..3d722aae5f15 100644
@@ -56,8 +56,11 @@ struct rxe_pool_entry {
 	struct kref		ref_cnt;
 	struct list_head	list;
 
-	/* only used if indexed or keyed */
-	struct rb_node		node;
+	/* only used if keyed */
+	struct rb_node		key_node;
+
+	/* only used if indexed */
+	struct rb_node		index_node;
 	u32			index;
 };
 
@@ -74,15 +77,22 @@ struct rxe_pool {
 	unsigned int		max_elem;
 	atomic_t		num_elem;
 
-	/* only used if indexed or keyed */
-	struct rb_root		tree;
-	unsigned long		*table;
-	size_t			table_size;
-	u32			max_index;
-	u32			min_index;
-	u32			last;
-	size_t			key_offset;
-	size_t			key_size;
+	/* only used if indexed */
+	struct {
+		struct rb_root		tree;
+		unsigned long		*table;
+		size_t			table_size;
+		u32			last;
+		u32			max_index;
+		u32			min_index;
+	} index;
+
+	/* only used if keyed */
+	struct {
+		struct rb_root		tree;
+		size_t			key_offset;
+		size_t			key_size;
+	} key;
 };
 
 /* initialize a pool of objects with given limit on
Signature · 1 line
2.25.1
#4 lore raw

PATCH v7 4/19 rdma_rxe: make pool return values position independent

In reply to Bob Pearson

rxe pool manages objects which are based on a base type of rxe_pool_entry. Once apon a time these were always the first entry in the rxe object structs. Now the pool entries (usually named elem or pelem) occur in more than one position. In order to make the return values from lookup and allocate routines work independently of the position of the pelem the offset of pelem is added to the rxe_type_info struct and used to compute the return value from these routines.

---
 drivers/infiniband/sw/rxe/rxe_mcast.c |  4 +--
 drivers/infiniband/sw/rxe/rxe_mr.c    |  6 ++--
 drivers/infiniband/sw/rxe/rxe_pool.c  | 40 +++++++++++++++++++++------
 drivers/infiniband/sw/rxe/rxe_pool.h  |  5 ++--
 drivers/infiniband/sw/rxe/rxe_recv.c  |  6 ++--
 drivers/infiniband/sw/rxe/rxe_req.c   |  2 +-
 drivers/infiniband/sw/rxe/rxe_resp.c  |  2 +-
 7 files changed, 44 insertions(+), 21 deletions(-)
drivers/infiniband/sw/rxe/rxe_mcast.c +2 -2
index c02315aed8d1..b09c6594045a 100644
@@ -18,7 +18,7 @@ int rxe_mcast_get_grp(struct rxe_dev *rxe, union ib_gid *mgid,
 		goto err1;
 	}
 
-	grp = rxe_pool_get_key(&rxe->mc_grp_pool, mgid);
+	grp = rxe_get_key(&rxe->mc_grp_pool, mgid);
 	if (grp)
 		goto done;
 
@@ -98,7 +98,7 @@ int rxe_mcast_drop_grp_elem(struct rxe_dev *rxe, struct rxe_qp *qp,
 	struct rxe_mc_grp *grp;
 	struct rxe_mc_elem *elem, *tmp;
 
-	grp = rxe_pool_get_key(&rxe->mc_grp_pool, mgid);
+	grp = rxe_get_key(&rxe->mc_grp_pool, mgid);
 	if (!grp)
 		goto err1;
 
drivers/infiniband/sw/rxe/rxe_mr.c +3 -3
index 4baa2f498e9d..f40cf4df394f 100644
@@ -10,7 +10,7 @@
 /*
  * lfsr (linear feedback shift register) with period 255
  */
-static u8 rxe_get_key(void)
+static u8 get_key(void)
 {
 	static u32 key = 1;
 
@@ -49,7 +49,7 @@ int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)
 
 static void rxe_mr_init(int access, struct rxe_mr *mr)
 {
-	u32 lkey = mr->pelem.index << 8 | rxe_get_key();
+	u32 lkey = mr->pelem.index << 8 | get_key();
 	u32 rkey = (access & IB_ACCESS_REMOTE) ? lkey : 0;
 
 	if (mr->pelem.pool->type == RXE_TYPE_MR) {
@@ -544,7 +544,7 @@ struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
 	struct rxe_dev *rxe = to_rdev(pd->ibpd.device);
 	int index = key >> 8;
 
-	mr = rxe_pool_get_index(&rxe->mr_pool, index);
+	mr = rxe_get_index(&rxe->mr_pool, index);
 	if (!mr)
 		return NULL;
 
drivers/infiniband/sw/rxe/rxe_pool.c +31 -9
index 30b8f037ee20..c4f1318ea761 100644
@@ -13,21 +13,25 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
 	[RXE_TYPE_UC] = {
 		.name		= "rxe-uc",
 		.size		= sizeof(struct rxe_ucontext),
+		.elem_offset	= offsetof(struct rxe_ucontext, pelem),
 		.flags          = RXE_POOL_NO_ALLOC,
 	},
 	[RXE_TYPE_PD] = {
 		.name		= "rxe-pd",
 		.size		= sizeof(struct rxe_pd),
+		.elem_offset	= offsetof(struct rxe_pd, pelem),
 		.flags		= RXE_POOL_NO_ALLOC,
 	},
 	[RXE_TYPE_AH] = {
 		.name		= "rxe-ah",
 		.size		= sizeof(struct rxe_ah),
+		.elem_offset	= offsetof(struct rxe_ah, pelem),
 		.flags		= RXE_POOL_ATOMIC | RXE_POOL_NO_ALLOC,
 	},
 	[RXE_TYPE_SRQ] = {
 		.name		= "rxe-srq",
 		.size		= sizeof(struct rxe_srq),
+		.elem_offset	= offsetof(struct rxe_srq, pelem),
 		.flags		= RXE_POOL_INDEX | RXE_POOL_NO_ALLOC,
 		.min_index	= RXE_MIN_SRQ_INDEX,
 		.max_index	= RXE_MAX_SRQ_INDEX,
@@ -35,6 +39,7 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
 	[RXE_TYPE_QP] = {
 		.name		= "rxe-qp",
 		.size		= sizeof(struct rxe_qp),
+		.elem_offset	= offsetof(struct rxe_qp, pelem),
 		.cleanup	= rxe_qp_cleanup,
 		.flags		= RXE_POOL_INDEX,
 		.min_index	= RXE_MIN_QP_INDEX,
@@ -43,12 +48,14 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
 	[RXE_TYPE_CQ] = {
 		.name		= "rxe-cq",
 		.size		= sizeof(struct rxe_cq),
+		.elem_offset	= offsetof(struct rxe_cq, pelem),
 		.flags          = RXE_POOL_NO_ALLOC,
 		.cleanup	= rxe_cq_cleanup,
 	},
 	[RXE_TYPE_MR] = {
 		.name		= "rxe-mr",
 		.size		= sizeof(struct rxe_mr),
+		.elem_offset	= offsetof(struct rxe_mr, pelem),
 		.cleanup	= rxe_mr_cleanup,
 		.flags		= RXE_POOL_INDEX,
 		.max_index	= RXE_MAX_MR_INDEX,
@@ -57,6 +64,7 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
 	[RXE_TYPE_MW] = {
 		.name		= "rxe-mw",
 		.size		= sizeof(struct rxe_mw),
+		.elem_offset	= offsetof(struct rxe_mw, pelem),
 		.flags		= RXE_POOL_INDEX,
 		.max_index	= RXE_MAX_MW_INDEX,
 		.min_index	= RXE_MIN_MW_INDEX,
@@ -64,6 +72,7 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
 	[RXE_TYPE_MC_GRP] = {
 		.name		= "rxe-mc_grp",
 		.size		= sizeof(struct rxe_mc_grp),
+		.elem_offset	= offsetof(struct rxe_mc_grp, pelem),
 		.cleanup	= rxe_mc_cleanup,
 		.flags		= RXE_POOL_KEY,
 		.key_offset	= offsetof(struct rxe_mc_grp, mgid),
@@ -72,6 +81,7 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
 	[RXE_TYPE_MC_ELEM] = {
 		.name		= "rxe-mc_elem",
 		.size		= sizeof(struct rxe_mc_elem),
+		.elem_offset	= offsetof(struct rxe_mc_elem, pelem),
 		.flags		= RXE_POOL_ATOMIC,
 	},
 };
@@ -304,6 +314,8 @@ void rxe_drop_index(void *arg)
 void *rxe_alloc(struct rxe_pool *pool)
 {
 	struct rxe_pool_entry *elem;
+	struct rxe_type_info *info = &rxe_type_info[pool->type];
+	u8 *obj = NULL;
 	unsigned long flags;
 
 	might_sleep_if(!(pool->flags & RXE_POOL_ATOMIC));
@@ -330,8 +342,9 @@ void *rxe_alloc(struct rxe_pool *pool)
 
 	elem->pool = pool;
 	kref_init(&elem->ref_cnt);
+	obj = (u8 *)elem - info->elem_offset;
 
-	return elem;
+	return obj;
 
 out_cnt:
 	atomic_dec(&pool->num_elem);
@@ -390,10 +403,12 @@ void rxe_elem_release(struct kref *kref)
 	rxe_pool_put(pool);
 }
 
-void *rxe_pool_get_index(struct rxe_pool *pool, u32 index)
+void *rxe_get_index(struct rxe_pool *pool, u32 index)
 {
 	struct rb_node *node = NULL;
 	struct rxe_pool_entry *elem = NULL;
+	struct rxe_type_info *info = &rxe_type_info[pool->type];
+	u8 *obj = NULL;
 	unsigned long flags;
 
 	read_lock_irqsave(&pool->pool_lock, flags);
@@ -410,21 +425,25 @@ void *rxe_pool_get_index(struct rxe_pool *pool, u32 index)
 			node = node->rb_left;
 		else if (elem->index < index)
 			node = node->rb_right;
-		else {
-			kref_get(&elem->ref_cnt);
+		else
 			break;
-		}
 	}
 
+	if (node) {
+		kref_get(&elem->ref_cnt);
+		obj = (u8 *)elem - info->elem_offset;
+	}
 out:
 	read_unlock_irqrestore(&pool->pool_lock, flags);
-	return node ? elem : NULL;
+	return obj;
 }
 
-void *rxe_pool_get_key(struct rxe_pool *pool, void *key)
+void *rxe_get_key(struct rxe_pool *pool, void *key)
 {
 	struct rb_node *node = NULL;
 	struct rxe_pool_entry *elem = NULL;
+	struct rxe_type_info *info = &rxe_type_info[pool->type];
+	u8 *obj = NULL;
 	int cmp;
 	unsigned long flags;
 
@@ -449,10 +468,13 @@ void *rxe_pool_get_key(struct rxe_pool *pool, void *key)
 			break;
 	}
 
-	if (node)
+	if (node) {
 		kref_get(&elem->ref_cnt);
+		obj = (u8 *)elem - info->elem_offset;
+	}
+
 
 out:
 	read_unlock_irqrestore(&pool->pool_lock, flags);
-	return node ? elem : NULL;
+	return obj;
 }
drivers/infiniband/sw/rxe/rxe_pool.h +3 -2
index 3d722aae5f15..68c6dbeb72d4 100644
@@ -36,6 +36,7 @@ struct rxe_pool_entry;
 struct rxe_type_info {
 	const char		*name;
 	size_t			size;
+	size_t			elem_offset;
 	void			(*cleanup)(struct rxe_pool_entry *obj);
 	enum rxe_pool_flags	flags;
 	u32			max_index;
@@ -128,10 +129,10 @@ void rxe_add_key(void *elem, void *key);
 void rxe_drop_key(void *elem);
 
 /* lookup an indexed object from index. takes a reference on object */
-void *rxe_pool_get_index(struct rxe_pool *pool, u32 index);
+void *rxe_get_index(struct rxe_pool *pool, u32 index);
 
 /* lookup keyed object from key. takes a reference on the object */
-void *rxe_pool_get_key(struct rxe_pool *pool, void *key);
+void *rxe_get_key(struct rxe_pool *pool, void *key);
 
 /* cleanup an object when all references are dropped */
 void rxe_elem_release(struct kref *kref);
drivers/infiniband/sw/rxe/rxe_recv.c +3 -3
index a3eed4da1540..e14cb20c4596 100644
@@ -185,7 +185,7 @@ static int hdr_check(struct rxe_pkt_info *pkt)
 	if (qpn != IB_MULTICAST_QPN) {
 		index = (qpn == 1) ? port->qp_gsi_index : qpn;
 
-		qp = rxe_pool_get_index(&rxe->qp_pool, index);
+		qp = rxe_get_index(&rxe->qp_pool, index);
 		if (unlikely(!qp)) {
 			pr_warn_ratelimited("no qp matches qpn 0x%x\n", qpn);
 			goto err1;
@@ -242,7 +242,7 @@ static void rxe_rcv_mcast_pkt(struct rxe_dev *rxe, struct sk_buff *skb)
 		memcpy(&dgid, &ipv6_hdr(skb)->daddr, sizeof(dgid));
 
 	/* lookup mcast group corresponding to mgid, takes a ref */
-	mcg = rxe_pool_get_key(&rxe->mc_grp_pool, &dgid);
+	mcg = rxe_get_key(&rxe->mc_grp_pool, &dgid);
 	if (!mcg)
 		goto err1;	/* mcast group not registered */
 
@@ -274,7 +274,7 @@ static void rxe_rcv_mcast_pkt(struct rxe_dev *rxe, struct sk_buff *skb)
 
 	spin_unlock_bh(&mcg->mcg_lock);
 
-	rxe_drop_ref(mcg);	/* drop ref from rxe_pool_get_key. */
+	rxe_drop_ref(mcg);	/* drop ref from rxe_get_key. */
 
 err1:
 	kfree_skb(skb);
drivers/infiniband/sw/rxe/rxe_req.c +1 -1
index 57236d8c2146..ea08464e461d 100644
@@ -599,7 +599,7 @@ int rxe_requester(void *arg)
 			struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
 			struct rxe_mr *rmr;
 
-			rmr = rxe_pool_get_index(&rxe->mr_pool,
+			rmr = rxe_get_index(&rxe->mr_pool,
 						 wqe->wr.ex.invalidate_rkey >> 8);
 			if (!rmr) {
 				pr_err("No mr for key %#x\n",
drivers/infiniband/sw/rxe/rxe_resp.c +1 -1
index 69867bf39cfb..9f96b4b706c3 100644
@@ -888,7 +888,7 @@ static enum resp_states do_complete(struct rxe_qp *qp,
 				wc->wc_flags |= IB_WC_WITH_INVALIDATE;
 				wc->ex.invalidate_rkey = ieth_rkey(pkt);
 
-				rmr = rxe_pool_get_index(&rxe->mr_pool,
+				rmr = rxe_get_index(&rxe->mr_pool,
 							 wc->ex.invalidate_rkey >> 8);
 				if (unlikely(!rmr)) {
 					pr_err("Bad rkey %#x invalidation\n",
Signature · 1 line
2.25.1
#5 lore raw

PATCH v7 5/19 rdma_rxe: remove void * parameters in pool APIs

In reply to Bob Pearson

More work to make rxe pools type safe.
Replaced routines that took void * as parameters and and replaced them with rxe_pool_entry *elem. Then added macros to convert objects to pool entries. This makes these routines type safe against moving the pool entries around in the objects.

---
 drivers/infiniband/sw/rxe/rxe_pool.c  | 14 +++++--------
 drivers/infiniband/sw/rxe/rxe_pool.h  | 29 ++++++++++++++-------------
 drivers/infiniband/sw/rxe/rxe_verbs.c | 10 ++++-----
 3 files changed, 25 insertions(+), 28 deletions(-)
drivers/infiniband/sw/rxe/rxe_pool.c +5 -9
index c4f1318ea761..e40a64616d79 100644
@@ -264,9 +264,8 @@ static void insert_key(struct rxe_pool *pool, struct rxe_pool_entry *new)
 	return;
 }
 
-void rxe_add_key(void *arg, void *key)
+void __rxe_add_key(struct rxe_pool_entry *elem, void *key)
 {
-	struct rxe_pool_entry *elem = arg;
 	struct rxe_pool *pool = elem->pool;
 	unsigned long flags;
 
@@ -276,9 +275,8 @@ void rxe_add_key(void *arg, void *key)
 	write_unlock_irqrestore(&pool->pool_lock, flags);
 }
 
-void rxe_drop_key(void *arg)
+void __rxe_drop_key(struct rxe_pool_entry *elem)
 {
-	struct rxe_pool_entry *elem = arg;
 	struct rxe_pool *pool = elem->pool;
 	unsigned long flags;
 
@@ -287,9 +285,8 @@ void rxe_drop_key(void *arg)
 	write_unlock_irqrestore(&pool->pool_lock, flags);
 }
 
-void rxe_add_index(void *arg)
+void __rxe_add_index(struct rxe_pool_entry *elem)
 {
-	struct rxe_pool_entry *elem = arg;
 	struct rxe_pool *pool = elem->pool;
 	unsigned long flags;
 
@@ -299,9 +296,8 @@ void rxe_add_index(void *arg)
 	write_unlock_irqrestore(&pool->pool_lock, flags);
 }
 
-void rxe_drop_index(void *arg)
+void __rxe_drop_index(struct rxe_pool_entry *elem)
 {
-	struct rxe_pool_entry *elem = arg;
 	struct rxe_pool *pool = elem->pool;
 	unsigned long flags;
 
@@ -354,7 +350,7 @@ void *rxe_alloc(struct rxe_pool *pool)
 	return NULL;
 }
 
-int rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_entry *elem)
+int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_entry *elem)
 {
 	unsigned long flags;
 
drivers/infiniband/sw/rxe/rxe_pool.h +15 -14
index 68c6dbeb72d4..2f9be92ef6f9 100644
@@ -109,24 +109,25 @@ void rxe_pool_cleanup(struct rxe_pool *pool);
 /* allocate an object from pool */
 void *rxe_alloc(struct rxe_pool *pool);
 
-/* connect already allocated object to pool */
-int rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_entry *elem);
+int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_entry *elem);
 
-/* assign an index to an indexed object and insert object into
- *  pool's rb tree
- */
-void rxe_add_index(void *elem);
+#define rxe_add_to_pool(pool, obj) __rxe_add_to_pool(pool, &(obj)->pelem)
 
-/* drop an index and remove object from rb tree */
-void rxe_drop_index(void *elem);
+void __rxe_add_index(struct rxe_pool_entry *elem);
 
-/* assign a key to a keyed object and insert object into
- *  pool's rb tree
- */
-void rxe_add_key(void *elem, void *key);
+#define rxe_add_index(obj) __rxe_add_index(&(obj)->pelem)
+
+void __rxe_drop_index(struct rxe_pool_entry *elem);
+
+#define rxe_drop_index(obj) __rxe_drop_index(&(obj)->pelem)
+
+void __rxe_add_key(struct rxe_pool_entry *elem, void *key);
+
+#define rxe_add_key(obj, key) __rxe_add_key(&(obj)->pelem, key)
+
+void __rxe_drop_key(struct rxe_pool_entry *elem);
 
-/* remove elem from rb tree */
-void rxe_drop_key(void *elem);
+#define rxe_drop_key(obj) __rxe_drop_key(&(obj)->pelem)
 
 /* lookup an indexed object from index. takes a reference on object */
 void *rxe_get_index(struct rxe_pool *pool, u32 index);
drivers/infiniband/sw/rxe/rxe_verbs.c +5 -5
index 664bcb025aae..db1d7f5ca2a7 100644
@@ -111,7 +111,7 @@ static int rxe_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
 	struct rxe_dev *rxe = to_rdev(uctx->device);
 	struct rxe_ucontext *uc = to_ruc(uctx);
 
-	return rxe_add_to_pool(&rxe->uc_pool, &uc->pelem);
+	return rxe_add_to_pool(&rxe->uc_pool, uc);
 }
 
 static void rxe_dealloc_ucontext(struct ib_ucontext *ibuc)
@@ -145,7 +145,7 @@ static int rxe_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 	struct rxe_dev *rxe = to_rdev(ibpd->device);
 	struct rxe_pd *pd = to_rpd(ibpd);
 
-	return rxe_add_to_pool(&rxe->pd_pool, &pd->pelem);
+	return rxe_add_to_pool(&rxe->pd_pool, pd);
 }
 
 static int rxe_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
@@ -169,7 +169,7 @@ static int rxe_create_ah(struct ib_ah *ibah,
 	if (err)
 		return err;
 
-	err = rxe_add_to_pool(&rxe->ah_pool, &ah->pelem);
+	err = rxe_add_to_pool(&rxe->ah_pool, ah);
 	if (err)
 		return err;
 
@@ -275,7 +275,7 @@ static int rxe_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init,
 	if (err)
 		goto err1;
 
-	err = rxe_add_to_pool(&rxe->srq_pool, &srq->pelem);
+	err = rxe_add_to_pool(&rxe->srq_pool, srq);
 	if (err)
 		goto err1;
 
@@ -776,7 +776,7 @@ static int rxe_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 	if (err)
 		return err;
 
-	return rxe_add_to_pool(&rxe->cq_pool, &cq->pelem);
+	return rxe_add_to_pool(&rxe->cq_pool, cq);
 }
 
 static int rxe_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
Signature · 1 line
2.25.1
#6 lore raw

PATCH v7 6/19 rdma_rxe: add alloc_mw and dealloc_mw verbs

In reply to Bob Pearson

v7:
- Adjust for mw being allocated in core.
- Made MW NO_ALLOC type in pool
- Rearranged rxe_mw struct to make ibmw first
v6:
- Add a new file focused on memory windows, rxe_mw.c.
- Add alloc_mw and dealloc_mw verbs and added them to the list of supported user space verbs.
- Added indices AND keys for MW and MR.
- Add indices to user api response for MW and MR
- Replaced lookup by index with lookup by key in rxe_req.c and rxe_resp.c

---
 drivers/infiniband/sw/rxe/Makefile    |  1 +
 drivers/infiniband/sw/rxe/rxe_loc.h   |  7 +++
 drivers/infiniband/sw/rxe/rxe_mr.c    | 80 +++++++++++--------------
 drivers/infiniband/sw/rxe/rxe_mw.c    | 85 +++++++++++++++++++++++++++
 drivers/infiniband/sw/rxe/rxe_pool.c  | 50 ++++++++++------
 drivers/infiniband/sw/rxe/rxe_pool.h  |  2 +-
 drivers/infiniband/sw/rxe/rxe_req.c   |  4 +-
 drivers/infiniband/sw/rxe/rxe_resp.c  |  4 +-
 drivers/infiniband/sw/rxe/rxe_verbs.c | 53 +++++++++++------
 drivers/infiniband/sw/rxe/rxe_verbs.h | 10 +++-
 include/uapi/rdma/rdma_user_rxe.h     | 10 ++++
 11 files changed, 219 insertions(+), 87 deletions(-)
drivers/infiniband/sw/rxe/Makefile +1 -0
index 66af72dca759..1e24673e9318 100644
@@ -15,6 +15,7 @@ rdma_rxe-y := \
 	rxe_qp.o \
 	rxe_cq.o \
 	rxe_mr.o \
+	rxe_mw.o \
 	rxe_opcode.o \
 	rxe_mmap.o \
 	rxe_icrc.o \
drivers/infiniband/sw/rxe/rxe_loc.h +7 -0
index 9ec6bff6863f..1502aa557621 100644
@@ -109,6 +109,13 @@ void rxe_mr_cleanup(struct rxe_pool_entry *arg);
 
 int advance_dma_data(struct rxe_dma_info *dma, unsigned int length);
 
+/* rxe_mw.c */
+int rxe_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata);
+
+int rxe_dealloc_mw(struct ib_mw *ibmw);
+
+void rxe_mw_cleanup(struct rxe_pool_entry *arg);
+
 /* rxe_net.c */
 void rxe_loopback(struct sk_buff *skb);
 int rxe_send(struct rxe_pkt_info *pkt, struct sk_buff *skb);
drivers/infiniband/sw/rxe/rxe_mr.c +35 -45

115 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_mw.c new file +85 -0

85 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_pool.c +32 -18

139 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_pool.h +1 -1
index 2f9be92ef6f9..e0e75faca96a 100644
@@ -121,7 +121,7 @@ void __rxe_drop_index(struct rxe_pool_entry *elem);
 
 #define rxe_drop_index(obj) __rxe_drop_index(&(obj)->pelem)
 
-void __rxe_add_key(struct rxe_pool_entry *elem, void *key);
+int __rxe_add_key(struct rxe_pool_entry *elem, void *key);
 
 #define rxe_add_key(obj, key) __rxe_add_key(&(obj)->pelem, key)
 
drivers/infiniband/sw/rxe/rxe_req.c +2 -2
index ea08464e461d..fe3fcc70411d 100644
@@ -599,8 +599,8 @@ int rxe_requester(void *arg)
 			struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
 			struct rxe_mr *rmr;
 
-			rmr = rxe_get_index(&rxe->mr_pool,
-						 wqe->wr.ex.invalidate_rkey >> 8);
+			rmr = rxe_get_key(&rxe->mr_pool,
+					    &wqe->wr.ex.invalidate_rkey);
 			if (!rmr) {
 				pr_err("No mr for key %#x\n",
 				       wqe->wr.ex.invalidate_rkey);
drivers/infiniband/sw/rxe/rxe_resp.c +2 -2
index 9f96b4b706c3..cc8a0bb7530b 100644
@@ -888,8 +888,8 @@ static enum resp_states do_complete(struct rxe_qp *qp,
 				wc->wc_flags |= IB_WC_WITH_INVALIDATE;
 				wc->ex.invalidate_rkey = ieth_rkey(pkt);
 
-				rmr = rxe_get_index(&rxe->mr_pool,
-							 wc->ex.invalidate_rkey >> 8);
+				rmr = rxe_get_key(&rxe->mr_pool,
+						  &wc->ex.invalidate_rkey);
 				if (unlikely(!rmr)) {
 					pr_err("Bad rkey %#x invalidation\n",
 					       wc->ex.invalidate_rkey);
drivers/infiniband/sw/rxe/rxe_verbs.c +35 -18

123 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_verbs.h +9 -1
index dbc649c9c43f..3253508a3b74 100644
@@ -319,9 +319,12 @@ struct rxe_mr {
 	struct rxe_map		**map;
 };
 
+/* use high order bit to separate MW and MR rkeys */
+#define IS_MW  (1 << 31)
+
 struct rxe_mw {
-	struct rxe_pool_entry	pelem;
 	struct ib_mw		ibmw;
+	struct rxe_pool_entry	pelem;
 	struct rxe_qp		*qp;	/* type 2B only */
 	struct rxe_mr		*mr;
 	spinlock_t		lock;
@@ -441,6 +444,11 @@ static inline struct rxe_mr *to_rmr(struct ib_mr *mr)
 	return mr ? container_of(mr, struct rxe_mr, ibmr) : NULL;
 }
 
+static inline struct rxe_mw *to_rmw(struct ib_mw *mw)
+{
+	return mw ? container_of(mw, struct rxe_mw, ibmw) : NULL;
+}
+
 int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name);
 
 void rxe_mc_cleanup(struct rxe_pool_entry *arg);
include/uapi/rdma/rdma_user_rxe.h +10 -0
index d8f2e0e46dab..4ad0fa0b2ab9 100644
@@ -175,4 +175,14 @@ struct rxe_modify_srq_cmd {
 	__aligned_u64 mmap_info_addr;
 };
 
+struct rxe_reg_mr_resp {
+	__u32 index;
+	__u32 reserved;
+};
+
+struct rxe_alloc_mw_resp {
+	__u32 index;
+	__u32 reserved;
+};
+
 #endif /* RDMA_USER_RXE_H */
Signature · 1 line
2.25.1
#7 lore raw

PATCH v7 7/19 rdma_rxe: Add bind_mw and invalidate_mw verbs

In reply to Bob Pearson

v7:
- adjust for new pool api changes
v6:
- Add code to implement ibv_bind_mw (for type 1 MWs) and post send queue bind_mw (for type 2 MWs).
- Add code to implement local (post send) and remote (send with invalidate) invalidate operations.
- Add rules checking for MW operations from IBA.

---
 drivers/infiniband/sw/rxe/rxe_comp.c   |   1 +
 drivers/infiniband/sw/rxe/rxe_loc.h    |   6 +
 drivers/infiniband/sw/rxe/rxe_mr.c     |  13 +-
 drivers/infiniband/sw/rxe/rxe_mw.c     | 289 ++++++++++++++++++++++++-
 drivers/infiniband/sw/rxe/rxe_opcode.c |  11 +-
 drivers/infiniband/sw/rxe/rxe_opcode.h |   1 -
 drivers/infiniband/sw/rxe/rxe_req.c    | 134 +++++++++---
 drivers/infiniband/sw/rxe/rxe_resp.c   |  81 +++++--
 drivers/infiniband/sw/rxe/rxe_verbs.c  |   2 +-
 drivers/infiniband/sw/rxe/rxe_verbs.h  |   7 +
 include/uapi/rdma/rdma_user_rxe.h      |  34 ++-
 11 files changed, 512 insertions(+), 67 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +1 -0
index 5dc86c9e74c2..8b81d3b24a8a 100644
@@ -103,6 +103,7 @@ static enum ib_wc_opcode wr_to_wc_opcode(enum ib_wr_opcode opcode)
 	case IB_WR_RDMA_READ_WITH_INV:		return IB_WC_RDMA_READ;
 	case IB_WR_LOCAL_INV:			return IB_WC_LOCAL_INV;
 	case IB_WR_REG_MR:			return IB_WC_REG_MR;
+	case IB_WR_BIND_MW:			return IB_WC_BIND_MW;
 
 	default:
 		return 0xff;
drivers/infiniband/sw/rxe/rxe_loc.h +6 -0
index 1502aa557621..0916fc8c541e 100644
@@ -109,6 +109,8 @@ void rxe_mr_cleanup(struct rxe_pool_entry *arg);
 
 int advance_dma_data(struct rxe_dma_info *dma, unsigned int length);
 
+int rxe_invalidate_mr(struct rxe_qp *qp, struct rxe_mr *mr);
+
 /* rxe_mw.c */
 int rxe_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata);
 
@@ -116,6 +118,10 @@ int rxe_dealloc_mw(struct ib_mw *ibmw);
 
 void rxe_mw_cleanup(struct rxe_pool_entry *arg);
 
+int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe);
+
+int rxe_invalidate_mw(struct rxe_qp *qp, struct rxe_mw *mw);
+
 /* rxe_net.c */
 void rxe_loopback(struct sk_buff *skb);
 int rxe_send(struct rxe_pkt_info *pkt, struct sk_buff *skb);
drivers/infiniband/sw/rxe/rxe_mr.c +12 -1
index af2d0d8877fe..a1aa5e2267d6 100644
@@ -532,12 +532,23 @@ struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
 	return mr;
 }
 
+int rxe_invalidate_mr(struct rxe_qp *qp, struct rxe_mr *mr)
+{
+	/* TODO there are API rules being ignored here
+	 * cleanup later. Current project is not trying
+	 * to fix MR
+	 */
+	mr->state = RXE_MEM_STATE_FREE;
+	return 0;
+}
+
 void rxe_mr_cleanup(struct rxe_pool_entry *elem)
 {
 	struct rxe_mr *mr = container_of(elem, typeof(*mr), pelem);
 	int i;
 
-	ib_umem_release(mr->umem);
+	if (mr->umem)
+		ib_umem_release(mr->umem);
 
 	if (mr->map) {
 		for (i = 0; i < mr->num_map; i++)
drivers/infiniband/sw/rxe/rxe_mw.c +282 -7

321 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_opcode.c +9 -2
index 0cb4b01fd910..5532f01ae5a3 100644
@@ -87,13 +87,20 @@ struct rxe_wr_opcode_info rxe_wr_opcode_info[] = {
 	[IB_WR_LOCAL_INV]				= {
 		.name	= "IB_WR_LOCAL_INV",
 		.mask	= {
-			[IB_QPT_RC]	= WR_REG_MASK,
+			[IB_QPT_RC]	= WR_LOCAL_MASK,
 		},
 	},
 	[IB_WR_REG_MR]					= {
 		.name	= "IB_WR_REG_MR",
 		.mask	= {
-			[IB_QPT_RC]	= WR_REG_MASK,
+			[IB_QPT_RC]	= WR_LOCAL_MASK,
+		},
+	},
+	[IB_WR_BIND_MW]					= {
+		.name	= "IB_WR_BIND_MW",
+		.mask	= {
+			[IB_QPT_RC]	= WR_LOCAL_MASK,
+			[IB_QPT_UC]	= WR_LOCAL_MASK,
 		},
 	},
 };
drivers/infiniband/sw/rxe/rxe_opcode.h +0 -1
index 1041ac9a9233..440e34f446bd 100644
@@ -20,7 +20,6 @@ enum rxe_wr_mask {
 	WR_READ_MASK			= BIT(3),
 	WR_WRITE_MASK			= BIT(4),
 	WR_LOCAL_MASK			= BIT(5),
-	WR_REG_MASK			= BIT(6),
 
 	WR_READ_OR_WRITE_MASK		= WR_READ_MASK | WR_WRITE_MASK,
 	WR_READ_WRITE_OR_SEND_MASK	= WR_READ_OR_WRITE_MASK | WR_SEND_MASK,
drivers/infiniband/sw/rxe/rxe_req.c +98 -36

207 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_resp.c +63 -18

128 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_verbs.c +1 -1
index f9425daad924..807c9a3b22ea 100644
@@ -577,7 +577,7 @@ static int init_send_wqe(struct rxe_qp *qp, const struct ib_send_wr *ibwr,
 
 			p += sge->length;
 		}
-	} else if (mask & WR_REG_MASK) {
+	} else if (mask & WR_LOCAL_MASK) {
 		wqe->mask = mask;
 		wqe->state = wqe_state_posted;
 		return 0;
drivers/infiniband/sw/rxe/rxe_verbs.h +7 -0
index 3253508a3b74..57849df8ccec 100644
@@ -316,9 +316,16 @@ struct rxe_mr {
 	u32			max_buf;
 	u32			num_map;
 
+	atomic_t		num_mw;
+
 	struct rxe_map		**map;
 };
 
+enum rxe_send_flags {
+	/* flag indicaes bind call came through verbs API */
+	RXE_BIND_MW		= (1 << 0),
+};
+
 /* use high order bit to separate MW and MR rkeys */
 #define IS_MW  (1 << 31)
 
include/uapi/rdma/rdma_user_rxe.h +33 -1
index 4ad0fa0b2ab9..d49125682359 100644
@@ -93,7 +93,39 @@ struct rxe_send_wr {
 			__u32	remote_qkey;
 			__u16	pkey_index;
 		} ud;
-		/* reg is only used by the kernel and is not part of the uapi */
+		struct {
+			__aligned_u64	addr;
+			__aligned_u64	length;
+			union {
+				__u32		mr_index;
+				__aligned_u64	reserved1;
+			};
+			union {
+				__u32		mw_index;
+				__aligned_u64	reserved2;
+			};
+			__u32	rkey;
+			__u32	access;
+			__u32	flags;
+		} umw;
+		/* The following are only used by the kernel
+		 * and are not part of the uapi
+		 */
+		struct {
+			__aligned_u64	addr;
+			__aligned_u64	length;
+			union {
+				struct ib_mr	*mr;
+				__aligned_u64	reserved1;
+			};
+			union {
+				struct ib_mw	*mw;
+				__aligned_u64	reserved2;
+			};
+			__u32	rkey;
+			__u32	access;
+			__u32	flags;
+		} kmw;
 		struct {
 			union {
 				struct ib_mr *mr;
Signature · 1 line
2.25.1
#8 lore raw

PATCH v7 8/19 rdma_rxe: Add memory access through MWs

In reply to Bob Pearson

v7:
- adjust for pool API changes
v6:
- Implement memory access through MWs.
- Add rules checks from IBA.

---
 drivers/infiniband/sw/rxe/rxe_loc.h   | 19 +++---
 drivers/infiniband/sw/rxe/rxe_mr.c    | 76 +++++++++++++++---------
 drivers/infiniband/sw/rxe/rxe_mw.c    | 57 +++++++++++++++---
 drivers/infiniband/sw/rxe/rxe_pool.c  | 14 +++--
 drivers/infiniband/sw/rxe/rxe_pool.h  |  3 +-
 drivers/infiniband/sw/rxe/rxe_req.c   | 15 ++---
 drivers/infiniband/sw/rxe/rxe_resp.c  | 83 ++++++++++++++++++++-------
 drivers/infiniband/sw/rxe/rxe_verbs.h |  1 +
 8 files changed, 189 insertions(+), 79 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +10 -9
index 0916fc8c541e..fcffd5075b18 100644
@@ -100,28 +100,29 @@ enum lookup_type {
 	lookup_remote,
 };
 
-struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
-			   enum lookup_type type);
-
-int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);
-
-void rxe_mr_cleanup(struct rxe_pool_entry *arg);
-
 int advance_dma_data(struct rxe_dma_info *dma, unsigned int length);
 
 int rxe_invalidate_mr(struct rxe_qp *qp, struct rxe_mr *mr);
 
+int rxe_mr_check_access(struct rxe_qp *qp, struct rxe_mr *mr,
+			int access, u64 va, u32 resid);
+
+void rxe_mr_cleanup(struct rxe_pool_entry *arg);
+
 /* rxe_mw.c */
 int rxe_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata);
 
 int rxe_dealloc_mw(struct ib_mw *ibmw);
 
-void rxe_mw_cleanup(struct rxe_pool_entry *arg);
-
 int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe);
 
 int rxe_invalidate_mw(struct rxe_qp *qp, struct rxe_mw *mw);
 
+int rxe_mw_check_access(struct rxe_qp *qp, struct rxe_mw *mw,
+			int access, u64 va, u32 resid);
+
+void rxe_mw_cleanup(struct rxe_pool_entry *arg);
+
 /* rxe_net.c */
 void rxe_loopback(struct sk_buff *skb);
 int rxe_send(struct rxe_pkt_info *pkt, struct sk_buff *skb);
drivers/infiniband/sw/rxe/rxe_mr.c +50 -26

112 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_mw.c +49 -8

75 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_pool.c +10 -4
index 43b4f91d86be..5b83b6088975 100644
@@ -72,6 +72,7 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
 		.name		= "rxe-mw",
 		.size		= sizeof(struct rxe_mw),
 		.elem_offset	= offsetof(struct rxe_mw, pelem),
+		.cleanup	= rxe_mw_cleanup,
 		.flags		= RXE_POOL_NO_ALLOC
 				| RXE_POOL_INDEX
 				| RXE_POOL_KEY,
@@ -220,11 +221,12 @@ static u32 alloc_index(struct rxe_pool *pool)
 	return index + pool->index.min_index;
 }
 
-static void insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new)
+static int insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new)
 {
 	struct rb_node **link = &pool->index.tree.rb_node;
 	struct rb_node *parent = NULL;
 	struct rxe_pool_entry *elem;
+	int ret = 0;
 
 	while (*link) {
 		parent = *link;
@@ -232,6 +234,7 @@ static void insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new)
 
 		if (elem->index == new->index) {
 			pr_warn("element already exists!\n");
+			ret = -EINVAL;
 			goto out;
 		}
 
@@ -244,7 +247,7 @@ static void insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new)
 	rb_link_node(&new->index_node, parent, link);
 	rb_insert_color(&new->index_node, &pool->index.tree);
 out:
-	return;
+	return ret;
 }
 
 static int insert_key(struct rxe_pool *pool, struct rxe_pool_entry *new)
@@ -302,15 +305,18 @@ void __rxe_drop_key(struct rxe_pool_entry *elem)
 	write_unlock_irqrestore(&pool->pool_lock, flags);
 }
 
-void __rxe_add_index(struct rxe_pool_entry *elem)
+int __rxe_add_index(struct rxe_pool_entry *elem)
 {
 	struct rxe_pool *pool = elem->pool;
 	unsigned long flags;
+	int ret;
 
 	write_lock_irqsave(&pool->pool_lock, flags);
 	elem->index = alloc_index(pool);
-	insert_index(pool, elem);
+	ret = insert_index(pool, elem);
 	write_unlock_irqrestore(&pool->pool_lock, flags);
+
+	return ret;
 }
 
 void __rxe_drop_index(struct rxe_pool_entry *elem)
drivers/infiniband/sw/rxe/rxe_pool.h +2 -1
index e0e75faca96a..5954fae6cf99 100644
@@ -113,7 +113,7 @@ int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_entry *elem);
 
 #define rxe_add_to_pool(pool, obj) __rxe_add_to_pool(pool, &(obj)->pelem)
 
-void __rxe_add_index(struct rxe_pool_entry *elem);
+int __rxe_add_index(struct rxe_pool_entry *elem);
 
 #define rxe_add_index(obj) __rxe_add_index(&(obj)->pelem)
 
@@ -144,4 +144,5 @@ void rxe_elem_release(struct kref *kref);
 /* drop a reference on an object */
 #define rxe_drop_ref(elem) kref_put(&(elem)->pelem.ref_cnt, rxe_elem_release)
 
+#define rxe_read_ref(elem) kref_read(&(elem)->pelem.ref_cnt)
 #endif /* RXE_POOL_H */
drivers/infiniband/sw/rxe/rxe_req.c +5 -10
index 7efafb3e21eb..0428965c664b 100644
@@ -645,7 +645,6 @@ int rxe_requester(void *arg)
 		case IB_WR_BIND_MW:
 			ret = rxe_bind_mw(qp, wqe);
 			if (ret) {
-				wqe->state = wqe_state_done;
 				wqe->status = IB_WC_MW_BIND_ERR;
 				goto err;
 			}
@@ -653,6 +652,7 @@ int rxe_requester(void *arg)
 		default:
 			pr_err_once("unexpected LOCAL WR opcode = %d\n",
 					wqe->wr.opcode);
+			wqe->status = IB_WC_LOC_QP_OP_ERR;
 			goto err;
 		}
 
@@ -698,13 +698,7 @@ int rxe_requester(void *arg)
 	payload = (mask & RXE_WRITE_OR_SEND) ? wqe->dma.resid : 0;
 	if (payload > mtu) {
 		if (qp_type(qp) == IB_QPT_UD) {
-			/* C10-93.1.1: If the total sum of all the buffer lengths specified for a
-			 * UD message exceeds the MTU of the port as returned by QueryHCA, the CI
-			 * shall not emit any packets for this message. Further, the CI shall not
-			 * generate an error due to this condition.
-			 */
-
-			/* fake a successful UD send */
+			/* C10-93.1.1: fake a successful UD send */
 			wqe->first_psn = qp->req.psn;
 			wqe->last_psn = qp->req.psn;
 			qp->req.psn = (qp->req.psn + 1) & BTH_PSN_MASK;
@@ -769,6 +763,8 @@ int rxe_requester(void *arg)
 	 * to be called again
 	 */
 	wqe->state = wqe_state_error;
+	qp->req.wqe_index = next_index(qp->sq.queue,
+				       qp->req.wqe_index);
 	__rxe_do_task(&qp->comp.task);
 	ret = -EAGAIN;
 	goto done;
@@ -784,8 +780,7 @@ int rxe_requester(void *arg)
 
 again:
 	/* we come here if we are done with the current wqe but want to
-	 * get called again. Mostly we loop back to next wqe so should
-	 * be all one way or the other
+	 * get called again.
 	 */
 	ret = 0;
 	goto done;
drivers/infiniband/sw/rxe/rxe_resp.c +62 -21

157 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_verbs.h +1 -0
index 57849df8ccec..e8d9c8388345 100644
@@ -183,6 +183,7 @@ struct rxe_resp_info {
 
 	/* RDMA read / atomic only */
 	u64			va;
+	u64			offset;
 	struct rxe_mr		*mr;
 	u32			resid;
 	u32			rkey;
Signature · 1 line
2.25.1
#9 lore raw

PATCH v7 9/19 rdma_rxe: Add locked and unlocked pool APIs

In reply to Bob Pearson

v7:
- Current object pool APIs use the pool_lock to protect concurrent execution from race conditions. Most APIs take the pool_lock, execute one operation and then free the lock.
This patch adds unlocked versions of the APIs so that a caller can take the pool_lock, combine sequences of operations and then free the lock. This will be required to fix some race conditions which occur without this fix.

---
 drivers/infiniband/sw/rxe/rxe_pool.c | 528 ++++++++++++++++++++-------
 drivers/infiniband/sw/rxe/rxe_pool.h | 111 +++---
 2 files changed, 467 insertions(+), 172 deletions(-)
drivers/infiniband/sw/rxe/rxe_pool.c +400 -128
index 5b83b6088975..9d53b4d71230 100644
@@ -7,9 +7,8 @@
 #include "rxe.h"
 #include "rxe_loc.h"
 
-/* info about object pools
- */
-struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
+/* info about object pools */
+static struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
 	[RXE_TYPE_UC] = {
 		.name		= "rxe-uc",
 		.size		= sizeof(struct rxe_ucontext),
@@ -62,8 +61,8 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
 		.flags		= RXE_POOL_ATOMIC
 				| RXE_POOL_INDEX
 				| RXE_POOL_KEY,
-		.max_index	= RXE_MAX_MR_INDEX,
 		.min_index	= RXE_MIN_MR_INDEX,
+		.max_index	= RXE_MAX_MR_INDEX,
 		.key_offset	= offsetof(struct rxe_mr, ibmr.lkey)
 				  - offsetof(struct rxe_mr, pelem),
 		.key_size	= sizeof(u32),
@@ -76,8 +75,8 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
 		.flags		= RXE_POOL_NO_ALLOC
 				| RXE_POOL_INDEX
 				| RXE_POOL_KEY,
-		.max_index	= RXE_MAX_MW_INDEX,
 		.min_index	= RXE_MIN_MW_INDEX,
+		.max_index	= RXE_MAX_MW_INDEX,
 		.key_offset	= offsetof(struct rxe_mw, ibmw.rkey)
 				  - offsetof(struct rxe_mw, pelem),
 		.key_size	= sizeof(u32),
@@ -106,21 +105,18 @@ static inline const char *pool_name(struct rxe_pool *pool)
 	return rxe_type_info[pool->type].name;
 }
 
-static int rxe_pool_init_index(struct rxe_pool *pool, u32 max, u32 min)
+static int init_index_table(struct rxe_pool *pool, u32 indices)
 {
 	int err = 0;
 	size_t size;
 
-	if ((max - min + 1) < pool->max_elem) {
+	if (indices < pool->max_elem) {
 		pr_warn("not enough indices for max_elem\n");
 		err = -EINVAL;
 		goto out;
 	}
 
-	pool->index.max_index = max;
-	pool->index.min_index = min;
-
-	size = BITS_TO_LONGS(max - min + 1) * sizeof(long);
+	size = BITS_TO_LONGS(indices) * sizeof(long);
 	pool->index.table = kmalloc(size, GFP_KERNEL);
 	if (!pool->index.table) {
 		err = -ENOMEM;
@@ -128,31 +124,37 @@ static int rxe_pool_init_index(struct rxe_pool *pool, u32 max, u32 min)
 	}
 
 	pool->index.table_size = size;
-	bitmap_zero(pool->index.table, max - min + 1);
-
+	bitmap_zero(pool->index.table, indices);
 out:
 	return err;
 }
 
-int rxe_pool_init(
-	struct rxe_dev		*rxe,
-	struct rxe_pool		*pool,
-	enum rxe_elem_type	type,
-	unsigned int		max_elem)
+/**
+ * rxe_pool_init - Initialize an object pool
+ * @rxe: The rxe device containing pool
+ * @pool: The pool to initialize
+ * @type: The object type held in pool
+ * @max_elem: The pool size in elements
+ *
+ * Initialize a pool of objects with given limit on
+ * number of elements. gets parameters from rxe_type_info
+ */
+int rxe_pool_init(struct rxe_dev *rxe, struct rxe_pool *pool,
+		  enum rxe_elem_type type, unsigned int max_elem)
 {
-	int			err = 0;
-	size_t			size = rxe_type_info[type].size;
+	struct rxe_type_info *info = &rxe_type_info[type];
+	int err = 0;
 
 	memset(pool, 0, sizeof(*pool));
 
 	pool->rxe		= rxe;
 	pool->type		= type;
 	pool->max_elem		= max_elem;
-	pool->elem_size		= ALIGN(size, RXE_POOL_ALIGN);
-	pool->flags		= rxe_type_info[type].flags;
+	pool->elem_size		= ALIGN(info->size, RXE_POOL_ALIGN);
+	pool->flags		= info->flags;
 	pool->index.tree	= RB_ROOT;
 	pool->key.tree		= RB_ROOT;
-	pool->cleanup		= rxe_type_info[type].cleanup;
+	pool->cleanup		= info->cleanup;
 
 	atomic_set(&pool->num_elem, 0);
 
@@ -160,26 +162,27 @@ int rxe_pool_init(
 
 	rwlock_init(&pool->pool_lock);
 
-	if (rxe_type_info[type].flags & RXE_POOL_INDEX) {
-		err = rxe_pool_init_index(pool,
-					  rxe_type_info[type].max_index,
-					  rxe_type_info[type].min_index);
+	if (info->flags & RXE_POOL_INDEX) {
+		pool->index.min_index = info->min_index;
+		pool->index.max_index = info->max_index;
+
+		err = init_index_table(pool, info->max_index
+					- info->min_index + 1);
 		if (err)
 			goto out;
 	}
 
-	if (rxe_type_info[type].flags & RXE_POOL_KEY) {
-		pool->key.key_offset = rxe_type_info[type].key_offset;
-		pool->key.key_size = rxe_type_info[type].key_size;
+	if (info->flags & RXE_POOL_KEY) {
+		pool->key.key_offset = info->key_offset;
+		pool->key.key_size = info->key_size;
 	}
 
 	pool->state = RXE_POOL_STATE_VALID;
-
 out:
 	return err;
 }
 
-static void rxe_pool_release(struct kref *kref)
+static void pool_release(struct kref *kref)
 {
 	struct rxe_pool *pool = container_of(kref, struct rxe_pool, ref_cnt);
 
@@ -187,11 +190,19 @@ static void rxe_pool_release(struct kref *kref)
 	kfree(pool->index.table);
 }
 
-static void rxe_pool_put(struct rxe_pool *pool)
-{
-	kref_put(&pool->ref_cnt, rxe_pool_release);
-}
-
+#define pool_get(pool) kref_get(&(pool)->ref_cnt)
+#define pool_put(pool) kref_put(&(pool)->ref_cnt, pool_release)
+
+/**
+ * rxe_pool_cleanup - Cleanup an object pool
+ * @pool: The pool to cleanup
+ *
+ * Marks pool as invalid and drops a ref to the pool.
+ * When the ref count gets to zero pool_release will
+ * delete the table memory held by the pool.
+ * Pool entries hold references to the pool so it will
+ * hang around until they are all removed.
+ */
 void rxe_pool_cleanup(struct rxe_pool *pool)
 {
 	unsigned long flags;
@@ -203,9 +214,15 @@ void rxe_pool_cleanup(struct rxe_pool *pool)
 			pool_name(pool));
 	write_unlock_irqrestore(&pool->pool_lock, flags);
 
-	rxe_pool_put(pool);
+	pool_put(pool);
 }
 
+/*
+ * Allocate a new index in the pool from the index bit table.
+ * Search from the last allocated index forward.
+ * Caller should hold pool->pool_lock to be thread safe.
+ * There should be more possible indices than objects in the pool.
+ */
 static u32 alloc_index(struct rxe_pool *pool)
 {
 	u32 index;
@@ -215,12 +232,21 @@ static u32 alloc_index(struct rxe_pool *pool)
 	if (index >= range)
 		index = find_first_zero_bit(pool->index.table, range);
 
-	WARN_ON_ONCE(index >= range);
+	if (unlikely(index >= range))
+		pr_err("Unable to allocate a new index in %s pool\n",
+				pool_name(pool));
+
 	set_bit(index, pool->index.table);
 	pool->index.last = index;
+
 	return index + pool->index.min_index;
 }
 
+/*
+ * Insert index in index-red-black tree in pool.
+ * Used to lookup object by index. Returns an error
+ * if index is already inserted.
+ */
 static int insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new)
 {
 	struct rb_node **link = &pool->index.tree.rb_node;
@@ -232,8 +258,9 @@ static int insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new)
 		parent = *link;
 		elem = rb_entry(parent, struct rxe_pool_entry, index_node);
 
-		if (elem->index == new->index) {
-			pr_warn("element already exists!\n");
+		if (unlikely(elem->index == new->index)) {
+			pr_warn("index %d already exists in %s pool\n",
+				new->index, pool_name(pool));
 			ret = -EINVAL;
 			goto out;
 		}
@@ -250,11 +277,79 @@ static int insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new)
 	return ret;
 }
 
+/**
+ * __add_index - Add index to pool entry
+ * @elem: pool entry to add index to
+ *
+ * Allocate a new index and insert into index-red-black tree.
+ * Caller should hold pool->pool_lock.
+ */
+void __add_index(struct rxe_pool_entry *elem)
+{
+	struct rxe_pool *pool = elem->pool;
+
+	elem->index = alloc_index(pool);
+	insert_index(pool, elem);
+}
+
+/**
+ * __rxe_add_index - Lock and add index to pool entry
+ * @elem: pool entry to add index to
+ *
+ * Allocate a new index and insert into index-red-black tree.
+ */
+void __rxe_add_index(struct rxe_pool_entry *elem)
+{
+	struct rxe_pool *pool = elem->pool;
+	unsigned long flags;
+
+	write_lock_irqsave(&pool->pool_lock, flags);
+	__add_index(elem);
+	write_unlock_irqrestore(&pool->pool_lock, flags);
+}
+
+/**
+ * __drop_index - Remove index from pool entry
+ * @elem: pool entry to remove index from
+ *
+ * Remove index from bit table and delete entry from index-red-black tree.
+ * Caller should hold pool->pool_lock.
+ */
+void __drop_index(struct rxe_pool_entry *elem)
+{
+	struct rxe_pool *pool = elem->pool;
+
+	clear_bit(elem->index - pool->index.min_index, pool->index.table);
+	rb_erase(&elem->index_node, &pool->index.tree);
+}
+
+/**
+ * __rxe_drop_index - Lock and remove index from pool entry
+ * @elem: pool entry to remove index from
+ *
+ * Remove index from bit table and delete from index-red-black tree.
+ */
+void __rxe_drop_index(struct rxe_pool_entry *elem)
+{
+	struct rxe_pool *pool = elem->pool;
+	unsigned long flags;
+
+	write_lock_irqsave(&pool->pool_lock, flags);
+	__drop_index(elem);
+	write_unlock_irqrestore(&pool->pool_lock, flags);
+}
+
+/*
+ * Insert key in key red-black tree in pool.
+ * Used to lookup object by key. Returns an error
+ * if key is already inserted.
+ */
 static int insert_key(struct rxe_pool *pool, struct rxe_pool_entry *new)
 {
 	struct rb_node **link = &pool->key.tree.rb_node;
 	struct rb_node *parent = NULL;
 	struct rxe_pool_entry *elem;
+	int ret = 0;
 	int cmp;
 
 	while (*link) {
@@ -262,11 +357,14 @@ static int insert_key(struct rxe_pool *pool, struct rxe_pool_entry *new)
 		elem = rb_entry(parent, struct rxe_pool_entry, key_node);
 
 		cmp = memcmp((u8 *)elem + pool->key.key_offset,
-			     (u8 *)new + pool->key.key_offset, pool->key.key_size);
+			     (u8 *)new + pool->key.key_offset,
+			     pool->key.key_size);
 
-		if (cmp == 0) {
-			pr_warn("key already exists!\n");
-			return -EAGAIN;
+		if (unlikely(cmp == 0)) {
+			pr_warn("key already exists in %s pool\n",
+				pool_name(pool));
+			ret = -EAGAIN;
+			goto out;
 		}
 
 		if (cmp > 0)
@@ -277,75 +375,98 @@ static int insert_key(struct rxe_pool *pool, struct rxe_pool_entry *new)
 
 	rb_link_node(&new->key_node, parent, link);
 	rb_insert_color(&new->key_node, &pool->key.tree);
-
-	return 0;
+out:
+	return ret;
 }
 
-int __rxe_add_key(struct rxe_pool_entry *elem, void *key)
+/**
+ * __add_key - Add key to pool entry
+ * @elem: pool entry to add index to
+ * @key: the key to add
+ *
+ * Copy key into entry and insert into key-red-black tree.
+ * Caller should hold pool->pool_lock.
+ */
+int __add_key(struct rxe_pool_entry *elem, void *key)
 {
-	struct rxe_pool *pool = elem->pool;
-	unsigned long flags;
 	int ret;
+	struct rxe_pool *pool = elem->pool;
 
-	write_lock_irqsave(&pool->pool_lock, flags);
 	memcpy((u8 *)elem + pool->key.key_offset, key, pool->key.key_size);
 	ret = insert_key(pool, elem);
-	write_unlock_irqrestore(&pool->pool_lock, flags);
 
 	return ret;
 }
 
-void __rxe_drop_key(struct rxe_pool_entry *elem)
+/**
+ * __rxe_add_key - Lock and add key to pool entry
+ * @elem: pool entry to add index to
+ * @key: the key to add
+ *
+ * Copy key into entry and insert into key-red-black tree.
+ */
+int __rxe_add_key(struct rxe_pool_entry *elem, void *key)
 {
+	int ret;
 	struct rxe_pool *pool = elem->pool;
 	unsigned long flags;
 
 	write_lock_irqsave(&pool->pool_lock, flags);
-	rb_erase(&elem->key_node, &pool->key.tree);
+	ret = __add_key(elem, key);
 	write_unlock_irqrestore(&pool->pool_lock, flags);
+
+	return ret;
 }
 
-int __rxe_add_index(struct rxe_pool_entry *elem)
+/**
+ * __drop_key - Remove key from pool entry
+ * @elem: pool entry to remove key from
+ *
+ * Delete entry from key-red-black tree.
+ * Caller should hold pool->pool_lock.
+ */
+void __drop_key(struct rxe_pool_entry *elem)
 {
 	struct rxe_pool *pool = elem->pool;
-	unsigned long flags;
-	int ret;
-
-	write_lock_irqsave(&pool->pool_lock, flags);
-	elem->index = alloc_index(pool);
-	ret = insert_index(pool, elem);
-	write_unlock_irqrestore(&pool->pool_lock, flags);
 
-	return ret;
+	rb_erase(&elem->key_node, &pool->key.tree);
 }
 
-void __rxe_drop_index(struct rxe_pool_entry *elem)
+/**
+ * __rxe_drop_key - Lock and remove key from pool entry
+ * @elem: pool entry to remove key from
+ *
+ * Delete entry from key-red-black tree.
+ */
+void __rxe_drop_key(struct rxe_pool_entry *elem)
 {
 	struct rxe_pool *pool = elem->pool;
 	unsigned long flags;
 
 	write_lock_irqsave(&pool->pool_lock, flags);
-	clear_bit(elem->index - pool->index.min_index, pool->index.table);
-	rb_erase(&elem->index_node, &pool->index.tree);
+	__drop_key(elem);
 	write_unlock_irqrestore(&pool->pool_lock, flags);
 }
 
-void *rxe_alloc(struct rxe_pool *pool)
+/**
+ * __add_to_pool - Add pool entry allocated elsewhere to pool
+ * @pool: Pool to add entry to
+ * @elem: Pool entry to add
+ *
+ * Used to add objects allocated in rdma-core to the pool.
+ * Entry holds a reference to pool to protect the pool pointer
+ * Takes a reference to the ib_device so it can't be deleted
+ * before the pool is deleted.
+ * Returns an error if pool has been deleted or if there are
+ * too many objects in the pool.
+ * Caller should hold the pool->pool_lock.
+ */
+int __add_to_pool(struct rxe_pool *pool, struct rxe_pool_entry *elem)
 {
-	struct rxe_pool_entry *elem;
-	struct rxe_type_info *info = &rxe_type_info[pool->type];
-	u8 *obj = NULL;
-	unsigned long flags;
+	if (pool->state != RXE_POOL_STATE_VALID)
+		return -EINVAL;
 
-	might_sleep_if(!(pool->flags & RXE_POOL_ATOMIC));
-
-	read_lock_irqsave(&pool->pool_lock, flags);
-	if (pool->state != RXE_POOL_STATE_VALID) {
-		read_unlock_irqrestore(&pool->pool_lock, flags);
-		return NULL;
-	}
-	kref_get(&pool->ref_cnt);
-	read_unlock_irqrestore(&pool->pool_lock, flags);
+	pool_get(pool);
 
 	if (!ib_device_try_get(&pool->rxe->ib_dev))
 		goto out_put_pool;
@@ -353,85 +474,153 @@ void *rxe_alloc(struct rxe_pool *pool)
 	if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
 		goto out_cnt;
 
-	elem = kzalloc(rxe_type_info[pool->type].size,
-				 (pool->flags & RXE_POOL_ATOMIC) ?
-				 GFP_ATOMIC : GFP_KERNEL);
-	if (!elem)
-		goto out_cnt;
-
 	elem->pool = pool;
 	kref_init(&elem->ref_cnt);
-	obj = (u8 *)elem - info->elem_offset;
-
-	return obj;
 
+	return 0;
 out_cnt:
 	atomic_dec(&pool->num_elem);
 	ib_device_put(&pool->rxe->ib_dev);
 out_put_pool:
-	rxe_pool_put(pool);
-	return NULL;
+	pool_put(pool);
+	return -EINVAL;
 }
 
+/**
+ * __rxe_add_to_pool - Lock and add pool entry allocated elsewhere to pool
+ * @pool: Pool to add entry to
+ * @elem: Pool entry to add
+ *
+ * Same as __add_to_pool except takes the pool_lock.
+ */
 int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_entry *elem)
 {
+	int err;
 	unsigned long flags;
 
-	read_lock_irqsave(&pool->pool_lock, flags);
+	write_lock_irqsave(&pool->pool_lock, flags);
+	err = __add_to_pool(pool, elem);
+	write_unlock_irqrestore(&pool->pool_lock, flags);
+
+	return err;
+}
+
+/**
+ * __alloc - Create a new pool entry
+ * @pool: Pool to create entry in
+ *
+ * Entry holds a reference to pool to protect the pool pointer
+ * Takes a reference to the ib_device so it can't be deleted
+ * before the pool is deleted.
+ * Returns an error pointer if pool has been deleted or if there are
+ * too many objects in the pool or it cannot get memory for the
+ * object.
+ * Caller should hold the pool->pool_lock.
+ */
+void *__alloc(struct rxe_pool *pool)
+{
+	struct rxe_type_info *info = &rxe_type_info[pool->type];
+	void *obj;
+	struct rxe_pool_entry *elem;
+	int ret = 0;
+
 	if (pool->state != RXE_POOL_STATE_VALID) {
-		read_unlock_irqrestore(&pool->pool_lock, flags);
-		return -EINVAL;
+		ret = -EINVAL;
+		goto out;
 	}
-	kref_get(&pool->ref_cnt);
-	read_unlock_irqrestore(&pool->pool_lock, flags);
 
-	if (!ib_device_try_get(&pool->rxe->ib_dev))
+	pool_get(pool);
+
+	if (!ib_device_try_get(&pool->rxe->ib_dev)) {
+		ret = -EINVAL;
 		goto out_put_pool;
+	}
 
-	if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
+	if (atomic_inc_return(&pool->num_elem) > pool->max_elem) {
+		ret = -EINVAL;
+		goto out_cnt;
+	}
+
+	obj = kzalloc(info->size, GFP_ATOMIC);
+	if (!obj) {
+		ret = -ENOMEM;
 		goto out_cnt;
+	}
+
+	elem = (struct rxe_pool_entry *)((u8 *)obj + info->elem_offset);
 
 	elem->pool = pool;
 	kref_init(&elem->ref_cnt);
 
-	return 0;
+	return obj;
 
 out_cnt:
 	atomic_dec(&pool->num_elem);
 	ib_device_put(&pool->rxe->ib_dev);
 out_put_pool:
-	rxe_pool_put(pool);
-	return -EINVAL;
+	pool_put(pool);
+out:
+	return ERR_PTR(ret);
 }
 
-void rxe_elem_release(struct kref *kref)
+/**
+ * rxe_alloc - Lock and create a new pool entry
+ * @pool: Pool to create entry in
+ *
+ * Same as __alloc except holds the pool_lock.
+ */
+void *rxe_alloc(struct rxe_pool *pool)
 {
-	struct rxe_pool_entry *elem =
-		container_of(kref, struct rxe_pool_entry, ref_cnt);
-	struct rxe_pool *pool = elem->pool;
+	void *obj;
+	struct rxe_pool_entry *elem;
+	unsigned long flags;
+	int err;
 
-	if (pool->cleanup)
-		pool->cleanup(elem);
+	/* allocate object outside of lock in case it sleeps */
+	obj = kzalloc(rxe_type_info[pool->type].size,
+			(pool->flags & RXE_POOL_ATOMIC) ?
+			GFP_ATOMIC : GFP_KERNEL);
+	if (!obj)
+		goto out;
 
-	if (!(pool->flags & RXE_POOL_NO_ALLOC))
-		kfree(elem);
-	atomic_dec(&pool->num_elem);
-	ib_device_put(&pool->rxe->ib_dev);
-	rxe_pool_put(pool);
+	elem = (struct rxe_pool_entry *)((u8 *)obj +
+			rxe_type_info[pool->type].elem_offset);
+
+	write_lock_irqsave(&pool->pool_lock, flags);
+	err = __add_to_pool(pool, elem);
+	write_unlock_irqrestore(&pool->pool_lock, flags);
+	if (err) {
+		kfree(obj);
+		obj = NULL;
+	}
+out:
+	return obj;
 }
 
-void *rxe_get_index(struct rxe_pool *pool, u32 index)
+/**
+ * __get_index - Lookup index in pool and return object
+ * @pool: pool to lookup index in
+ * @index: index to lookup
+ *
+ * Lookup an index in pool's index-red-black tree and retrieve
+ * pool entry with that index.
+ * Convert entry address to object address with elem_offset.
+ * Returns the object address if found else NULL if not or
+ * an error pointer if the pool has been destroyed.
+ * Caller should hold the pool->pool_lock.
+ */
+void *__get_index(struct rxe_pool *pool, u32 index)
 {
-	struct rb_node *node = NULL;
-	struct rxe_pool_entry *elem = NULL;
 	struct rxe_type_info *info = &rxe_type_info[pool->type];
-	u8 *obj = NULL;
-	unsigned long flags;
-
-	read_lock_irqsave(&pool->pool_lock, flags);
+	struct rb_node *node;
+	struct rxe_pool_entry *elem = NULL;
+	void *obj;
+	int ret = 0;
 
-	if (pool->state != RXE_POOL_STATE_VALID)
+	if (pool->state != RXE_POOL_STATE_VALID) {
+		ret = -EINVAL;
 		goto out;
+	}
 
 	node = pool->index.tree.rb_node;
 
@@ -449,25 +638,59 @@ void *rxe_get_index(struct rxe_pool *pool, u32 index)
 	if (node) {
 		kref_get(&elem->ref_cnt);
 		obj = (u8 *)elem - info->elem_offset;
+	} else {
+		obj = NULL;
 	}
+
+	return obj;
 out:
+	return ERR_PTR(ret);
+}
+
+/**
+ * rxe_get_index - Lock and lookup index in pool and return object
+ * @pool: pool to lookup index in
+ * @index: index to lookup
+ *
+ * Same as __get_index except holds pool_lock.
+ */
+void *rxe_get_index(struct rxe_pool *pool, u32 index)
+{
+	void *obj;
+	unsigned long flags;
+
+	read_lock_irqsave(&pool->pool_lock, flags);
+	obj = __get_index(pool, index);
 	read_unlock_irqrestore(&pool->pool_lock, flags);
+
 	return obj;
 }
 
-void *rxe_get_key(struct rxe_pool *pool, void *key)
+/**
+ * __get_key - Lookup key in pool and return object
+ * @pool: pool to lookup key in
+ * @key: key to lookup
+ *
+ * Lookup a key in pool's key-red-black tree and retrieve
+ * pool entry with that key.
+ * Convert entry address to object address with elem_offset.
+ * Returns the object address if found else NULL if not or
+ * an error pointer if the pool has been destroyed.
+ * Caller should hold the pool->pool_lock.
+ */
+void *__get_key(struct rxe_pool *pool, void *key)
 {
-	struct rb_node *node = NULL;
-	struct rxe_pool_entry *elem = NULL;
 	struct rxe_type_info *info = &rxe_type_info[pool->type];
-	u8 *obj = NULL;
+	struct rb_node *node;
+	struct rxe_pool_entry *elem = NULL;
 	int cmp;
-	unsigned long flags;
-
-	read_lock_irqsave(&pool->pool_lock, flags);
+	void *obj;
+	int ret = 0;
 
-	if (pool->state != RXE_POOL_STATE_VALID)
+	if (pool->state != RXE_POOL_STATE_VALID) {
+		ret = -EINVAL;
 		goto out;
+	}
 
 	node = pool->key.tree.rb_node;
 
@@ -488,9 +711,58 @@ void *rxe_get_key(struct rxe_pool *pool, void *key)
 	if (node) {
 		kref_get(&elem->ref_cnt);
 		obj = (u8 *)elem - info->elem_offset;
+	} else {
+		obj = NULL;
 	}
 
+	return obj;
 out:
+	return ERR_PTR(ret);
+}
+
+/**
+ * rxe_get_key - Lock and lookup key in pool and return object
+ * @pool: pool to lookup key in
+ * @key: key to lookup
+ *
+ * Same as __get_key except holds pool_lock.
+ */
+void *rxe_get_key(struct rxe_pool *pool, void *key)
+{
+	void *obj;
+	unsigned long flags;
+
+	read_lock_irqsave(&pool->pool_lock, flags);
+	obj = __get_key(pool, key);
 	read_unlock_irqrestore(&pool->pool_lock, flags);
+
 	return obj;
 }
+
+/**
+ * rxe_elem_release - Delete pool object when ref count goes to zero
+ * @kref: Reference counter contained in pool entry
+ *
+ * If pool has an object cleanup routine call it.
+ * If entry memory was allocated in pool free it.
+ */
+void rxe_elem_release(struct kref *kref)
+{
+	struct rxe_pool_entry *elem = container_of(kref,
+					struct rxe_pool_entry, ref_cnt);
+	struct rxe_pool *pool = elem->pool;
+	struct rxe_type_info *info = &rxe_type_info[pool->type];
+	void *obj;
+
+	if (pool->cleanup)
+		pool->cleanup(elem);
+
+	if (!(pool->flags & RXE_POOL_NO_ALLOC)) {
+		obj = (u8 *)elem - info->elem_offset;
+		kfree(obj);
+	}
+
+	atomic_dec(&pool->num_elem);
+	ib_device_put(&pool->rxe->ib_dev);
+	pool_put(pool);
+}
drivers/infiniband/sw/rxe/rxe_pool.h +67 -44

184 lines not on this page · raw message

Signature · 1 line
2.25.1
#10 lore raw

PATCH v7 10/19 rdma_rxe: Add support for ibv_query_device_ex

In reply to Bob Pearson

Add code to initialize new struct members in
ib_device_attr as place holders.

---
 drivers/infiniband/sw/rxe/rxe.c       | 101 ++++++++++++++++++--------
 drivers/infiniband/sw/rxe/rxe_verbs.c |   7 +-
 2 files changed, 75 insertions(+), 33 deletions(-)
drivers/infiniband/sw/rxe/rxe.c +69 -32
index 8e0f9c489cab..ecc61f960c58 100644
@@ -40,40 +40,77 @@ void rxe_dealloc(struct ib_device *ib_dev)
 /* initialize rxe device parameters */
 static void rxe_init_device_param(struct rxe_dev *rxe)
 {
-	rxe->max_inline_data			= RXE_MAX_INLINE_DATA;
-
-	rxe->attr.vendor_id			= RXE_VENDOR_ID;
-	rxe->attr.max_mr_size			= RXE_MAX_MR_SIZE;
-	rxe->attr.page_size_cap			= RXE_PAGE_SIZE_CAP;
-	rxe->attr.max_qp			= RXE_MAX_QP;
-	rxe->attr.max_qp_wr			= RXE_MAX_QP_WR;
-	rxe->attr.device_cap_flags		= RXE_DEVICE_CAP_FLAGS;
-	rxe->attr.max_send_sge			= RXE_MAX_SGE;
-	rxe->attr.max_recv_sge			= RXE_MAX_SGE;
-	rxe->attr.max_sge_rd			= RXE_MAX_SGE_RD;
-	rxe->attr.max_cq			= RXE_MAX_CQ;
-	rxe->attr.max_cqe			= (1 << RXE_MAX_LOG_CQE) - 1;
-	rxe->attr.max_mr			= RXE_MAX_MR;
-	rxe->attr.max_mw			= RXE_MAX_MW;
-	rxe->attr.max_pd			= RXE_MAX_PD;
-	rxe->attr.max_qp_rd_atom		= RXE_MAX_QP_RD_ATOM;
-	rxe->attr.max_res_rd_atom		= RXE_MAX_RES_RD_ATOM;
-	rxe->attr.max_qp_init_rd_atom		= RXE_MAX_QP_INIT_RD_ATOM;
-	rxe->attr.atomic_cap			= IB_ATOMIC_HCA;
-	rxe->attr.max_mcast_grp			= RXE_MAX_MCAST_GRP;
-	rxe->attr.max_mcast_qp_attach		= RXE_MAX_MCAST_QP_ATTACH;
-	rxe->attr.max_total_mcast_qp_attach	= RXE_MAX_TOT_MCAST_QP_ATTACH;
-	rxe->attr.max_ah			= RXE_MAX_AH;
-	rxe->attr.max_srq			= RXE_MAX_SRQ;
-	rxe->attr.max_srq_wr			= RXE_MAX_SRQ_WR;
-	rxe->attr.max_srq_sge			= RXE_MAX_SRQ_SGE;
-	rxe->attr.max_fast_reg_page_list_len	= RXE_MAX_FMR_PAGE_LIST_LEN;
-	rxe->attr.max_pkeys			= RXE_MAX_PKEYS;
-	rxe->attr.local_ca_ack_delay		= RXE_LOCAL_CA_ACK_DELAY;
-	addrconf_addr_eui48((unsigned char *)&rxe->attr.sys_image_guid,
-			rxe->ndev->dev_addr);
+	struct ib_device_attr *a = &rxe->attr;
 
+	rxe->max_inline_data			= RXE_MAX_INLINE_DATA;
 	rxe->max_ucontext			= RXE_MAX_UCONTEXT;
+
+	a->atomic_cap				= IB_ATOMIC_HCA;
+	a->cq_caps.max_cq_moderation_count	= 0;
+	a->cq_caps.max_cq_moderation_period	= 0;
+	a->device_cap_flags			= RXE_DEVICE_CAP_FLAGS;
+	a->fw_ver				= 0;
+	a->hca_core_clock			= 0;
+	a->hw_ver				= 0;
+	a->local_ca_ack_delay			= RXE_LOCAL_CA_ACK_DELAY;
+	a->masked_atomic_cap			= 0;
+	a->max_ah				= RXE_MAX_AH;
+	a->max_cqe				= (1 << RXE_MAX_LOG_CQE) - 1;
+	a->max_cq				= RXE_MAX_CQ;
+	a->max_dm_size				= 0;
+	a->max_ee_init_rd_atom			= 0;
+	a->max_ee				= 0;
+	a->max_ee_rd_atom			= 0;
+	a->max_fast_reg_page_list_len		= RXE_MAX_FMR_PAGE_LIST_LEN;
+	a->max_mcast_grp			= RXE_MAX_MCAST_GRP;
+	a->max_mcast_qp_attach			= RXE_MAX_MCAST_QP_ATTACH;
+	a->max_mr				= RXE_MAX_MR;
+	a->max_mr_size				= RXE_MAX_MR_SIZE;
+	a->max_mw				= RXE_MAX_MW;
+	a->max_pd				= RXE_MAX_PD;
+	a->max_pi_fast_reg_page_list_len	= 0;
+	a->max_pkeys				= RXE_MAX_PKEYS;
+	a->max_qp_init_rd_atom			= RXE_MAX_QP_INIT_RD_ATOM;
+	a->max_qp_rd_atom			= RXE_MAX_QP_RD_ATOM;
+	a->max_qp				= RXE_MAX_QP;
+	a->max_qp_wr				= RXE_MAX_QP_WR;
+	a->max_raw_ethy_qp			= 0;
+	a->max_raw_ipv6_qp			= 0;
+	a->max_rdd				= 0;
+	a->max_recv_sge				= RXE_MAX_SGE;
+	a->max_res_rd_atom			= RXE_MAX_RES_RD_ATOM;
+	a->max_send_sge				= RXE_MAX_SGE;
+	a->max_sge_rd				= RXE_MAX_SGE_RD;
+	a->max_sgl_rd				= 0;
+	a->max_srq				= RXE_MAX_SRQ;
+	a->max_srq_sge				= RXE_MAX_SRQ_SGE;
+	a->max_srq_wr				= RXE_MAX_SRQ_WR;
+	a->max_total_mcast_qp_attach		= RXE_MAX_TOT_MCAST_QP_ATTACH;
+	a->max_wq_type_rq			= 0;
+	a->odp_caps.general_caps		= 0;
+	a->odp_caps.per_transport_caps.rc_odp_caps = 0;
+	a->odp_caps.per_transport_caps.uc_odp_caps = 0;
+	a->odp_caps.per_transport_caps.ud_odp_caps = 0;
+	a->odp_caps.per_transport_caps.xrc_odp_caps = 0;
+	a->page_size_cap			= RXE_PAGE_SIZE_CAP;
+	a->raw_packet_caps			= 0;
+	a->rss_caps.supported_qpts		= 0;
+	a->rss_caps.max_rwq_indirection_tables	= 0;
+	a->rss_caps.max_rwq_indirection_table_size = 0;
+	a->sig_guard_cap			= 0;
+	a->sig_prot_cap				= 0;
+	a->sys_image_guid			= 0;
+	a->timestamp_mask			= 0;
+	a->tm_caps.max_rndv_hdr_size		= 0;
+	a->tm_caps.max_num_tags			= 0;
+	a->tm_caps.flags			= 0;
+	a->tm_caps.max_ops			= 0;
+	a->tm_caps.max_sge			= 0;
+	a->vendor_id				= RXE_VENDOR_ID;
+	a->vendor_part_id			= 0;
+
+	addrconf_addr_eui48((unsigned char *)&a->sys_image_guid,
+			    rxe->ndev->dev_addr);
 }
 
 /* initialize port attributes */
drivers/infiniband/sw/rxe/rxe_verbs.c +6 -1
index 807c9a3b22ea..2695b286cd8e 100644
@@ -1150,7 +1150,8 @@ int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name)
 	dma_coerce_mask_and_coherent(&dev->dev,
 				     dma_get_required_mask(&dev->dev));
 
-	dev->uverbs_cmd_mask = BIT_ULL(IB_USER_VERBS_CMD_GET_CONTEXT)
+	dev->uverbs_cmd_mask =
+	      BIT_ULL(IB_USER_VERBS_CMD_GET_CONTEXT)
 	    | BIT_ULL(IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL)
 	    | BIT_ULL(IB_USER_VERBS_CMD_QUERY_DEVICE)
 	    | BIT_ULL(IB_USER_VERBS_CMD_QUERY_PORT)
@@ -1185,6 +1186,10 @@ int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name)
 	    | BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_MW)
 	    ;
 
+	dev->uverbs_ex_cmd_mask =
+	      BIT_ULL(IB_USER_VERBS_EX_CMD_QUERY_DEVICE)
+	    ;
+
 	ib_set_device_ops(dev, &rxe_dev_ops);
 	err = ib_device_set_netdev(&rxe->ib_dev, rxe->ndev, 1);
 	if (err)
Signature · 1 line
2.25.1
maintainer
#11 lore raw

Re: [PATCH for-next v7 10/19] rdma_rxe: Add support for ibv_query_device_ex

Quoted 8 lines · Bob Pearson

Add code to initialize new struct members in
ib_device_attr as place holders.

---
 drivers/infiniband/sw/rxe/rxe.c       | 101 ++++++++++++++++++--------
 drivers/infiniband/sw/rxe/rxe_verbs.c |   7 +-
 2 files changed, 75 insertions(+), 33 deletions(-)

This series should eliminate this patch and notably change the others

https://patchwork.kernel.org/project/linux-rdma/list/?series=359361

Can you take a look that it works for this?

Thanks,
Jason

#12 lore raw

Re: [PATCH for-next v7 10/19] rdma_rxe: Add support for ibv_query_device_ex

Quoted 10 lines · Jason Gunthorpe
Quoted 8 lines · Bob Pearson

Add code to initialize new struct members in
ib_device_attr as place holders.

---
 drivers/infiniband/sw/rxe/rxe.c       | 101 ++++++++++++++++++--------
 drivers/infiniband/sw/rxe/rxe_verbs.c |   7 +-
 2 files changed, 75 insertions(+), 33 deletions(-)
Quoted 9 lines

This series should eliminate this patch and notably change the others

https://patchwork.kernel.org/project/linux-rdma/list/?series=359361

Can you take a look that it works for this?

Thanks,
Jason

I'll take a look. Let me know if/when you take it into for-next. Should be easy to adjust for these. Currently I am working to try to improve performance. I have replaced tasklets with work queues. Low QP count performance is not changed much but it should help high QP count performance. At the moment I am trying to optimize UD since it is a lot simpler. Once that is done I want to look at core affinity.

Bob

#13 lore raw

PATCH v7 11/19 rdma_rxe: Add support for extended CQ operations

In reply to Bob Pearson

Add private members to user/kernel wc struct to carry extensions used by cq_ex.
Add timestamps on completion.
Add ignore overrun support.
Add commands to user API bitmasks.

---
 drivers/infiniband/sw/rxe/rxe_comp.c  |  7 ++++-
 drivers/infiniband/sw/rxe/rxe_resp.c  |  8 +++++-
 drivers/infiniband/sw/rxe/rxe_verbs.c | 10 ++++---
 drivers/infiniband/sw/rxe/rxe_verbs.h |  3 ++-
 include/uapi/rdma/rdma_user_rxe.h     | 38 ++++++++++++++++++++++-----
 5 files changed, 52 insertions(+), 14 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +6 -1
index 8b81d3b24a8a..72745ffcf118 100644
@@ -390,7 +390,7 @@ static void make_send_cqe(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
 		wc->byte_len		= wqe->dma.length;
 		wc->qp			= &qp->ibqp;
 	} else {
-		struct ib_uverbs_wc	*uwc	= &cqe->uibwc;
+		struct rxe_uverbs_wc	*uwc = &cqe->ruwc;
 
 		uwc->wr_id		= wqe->wr.wr_id;
 		uwc->status		= wqe->status;
@@ -400,6 +400,11 @@ static void make_send_cqe(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
 			uwc->wc_flags = IB_WC_WITH_IMM;
 		uwc->byte_len		= wqe->dma.length;
 		uwc->qp_num		= qp->ibqp.qp_num;
+		if (qp->scq->flags &
+		    IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION) {
+			uwc->timestamp	= (u64)ktime_get();
+			uwc->realtime	= (u64)ktime_get_real();
+		}
 	}
 }
 
drivers/infiniband/sw/rxe/rxe_resp.c +7 -1
index 2aae311f9063..58d3783063bb 100644
@@ -887,7 +887,7 @@ static enum resp_states do_complete(struct rxe_qp *qp,
 {
 	struct rxe_cqe cqe;
 	struct ib_wc *wc = &cqe.ibwc;
-	struct ib_uverbs_wc *uwc = &cqe.uibwc;
+	struct rxe_uverbs_wc *uwc = &cqe.ruwc;
 	struct rxe_recv_wqe *wqe = qp->resp.wqe;
 	struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
 
@@ -943,6 +943,12 @@ static enum resp_states do_complete(struct rxe_qp *qp,
 				uwc->src_qp = deth_sqp(pkt);
 
 			uwc->port_num		= qp->attr.port_num;
+
+			if (qp->rcq->flags &
+			    IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION) {
+				uwc->timestamp	= (u64)ktime_get();
+				uwc->realtime	= (u64)ktime_get_real();
+			}
 		} else {
 			struct sk_buff *skb = PKT_TO_SKB(pkt);
 
drivers/infiniband/sw/rxe/rxe_verbs.c +6 -4
index 2695b286cd8e..6be0aa9cf1f3 100644
@@ -749,7 +749,8 @@ static int rxe_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
 	return err;
 }
 
-static int rxe_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
+static int rxe_create_cq(struct ib_cq *ibcq,
+			 const struct ib_cq_init_attr *attr,
 			 struct ib_udata *udata)
 {
 	int err;
@@ -764,13 +765,12 @@ static int rxe_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 		uresp = udata->outbuf;
 	}
 
-	if (attr->flags)
-		return -EINVAL;
-
 	err = rxe_cq_chk_attr(rxe, NULL, attr->cqe, attr->comp_vector);
 	if (err)
 		return err;
 
+	cq->flags = attr->flags;
+
 	err = rxe_cq_from_init(rxe, cq, attr->cqe, attr->comp_vector, udata,
 			       uresp);
 	if (err)
@@ -1188,6 +1188,8 @@ int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name)
 
 	dev->uverbs_ex_cmd_mask =
 	      BIT_ULL(IB_USER_VERBS_EX_CMD_QUERY_DEVICE)
+	    | BIT_ULL(IB_USER_VERBS_EX_CMD_CREATE_CQ)
+	    | BIT_ULL(IB_USER_VERBS_EX_CMD_MODIFY_CQ)
 	    ;
 
 	ib_set_device_ops(dev, &rxe_dev_ops);
drivers/infiniband/sw/rxe/rxe_verbs.h +2 -1
index e8d9c8388345..54429d8a06fb 100644
@@ -53,7 +53,7 @@ struct rxe_ah {
 struct rxe_cqe {
 	union {
 		struct ib_wc		ibwc;
-		struct ib_uverbs_wc	uibwc;
+		struct rxe_uverbs_wc	ruwc;
 	};
 };
 
@@ -62,6 +62,7 @@ struct rxe_cq {
 	struct rxe_pool_entry	pelem;
 	struct rxe_queue	*queue;
 	spinlock_t		cq_lock;
+	u32			flags;
 	u8			notify;
 	bool			is_dying;
 	int			is_user;
include/uapi/rdma/rdma_user_rxe.h +31 -7
index d49125682359..95352e050ab4 100644
@@ -98,29 +98,27 @@ struct rxe_send_wr {
 			__aligned_u64	length;
 			union {
 				__u32		mr_index;
-				__aligned_u64	reserved1;
+				__aligned_u64   pad1;
 			};
 			union {
 				__u32		mw_index;
-				__aligned_u64	reserved2;
+				__aligned_u64   pad2;
 			};
 			__u32	rkey;
 			__u32	access;
 			__u32	flags;
 		} umw;
-		/* The following are only used by the kernel
-		 * and are not part of the uapi
-		 */
+		/* below are only used by the kernel */
 		struct {
 			__aligned_u64	addr;
 			__aligned_u64	length;
 			union {
 				struct ib_mr	*mr;
-				__aligned_u64	reserved1;
+				__aligned_u64   reserved1;
 			};
 			union {
 				struct ib_mw	*mw;
-				__aligned_u64	reserved2;
+				__aligned_u64   reserved2;
 			};
 			__u32	rkey;
 			__u32	access;
@@ -184,6 +182,32 @@ struct rxe_recv_wqe {
 	struct rxe_dma_info	dma;
 };
 
+struct rxe_uverbs_wc {
+	/* keep these the same as ib_uverbs_wc */
+	__aligned_u64		wr_id;
+	__u32			status;
+	__u32			opcode;
+	__u32			vendor_err;
+	__u32			byte_len;
+	union {
+		__be32		imm_data;
+		__u32		invalidate_rkey;
+	} ex;
+	__u32			qp_num;
+	__u32			src_qp;
+	__u32			wc_flags;
+	__u16			pkey_index;
+	__u16			slid;
+	__u8			sl;
+	__u8			dlid_path_bits;
+	__u8			port_num;
+	__u8			reserved;
+
+	/* any extras go here */
+	__aligned_u64		timestamp;
+	__aligned_u64		realtime;
+};
+
 struct rxe_create_cq_resp {
 	struct mminfo mi;
 };
Signature · 1 line
2.25.1
#14 lore raw

PATCH v7 12/19 rdma_rxe: Add support for extended QP operations

In reply to Bob Pearson

Add bits to user api command bitmask.

---
 drivers/infiniband/sw/rxe/rxe_verbs.c | 2 ++
 1 file changed, 2 insertions(+)
drivers/infiniband/sw/rxe/rxe_verbs.c +2 -0
index 6be0aa9cf1f3..4de263e41b07 100644
@@ -1188,6 +1188,8 @@ int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name)
 
 	dev->uverbs_ex_cmd_mask =
 	      BIT_ULL(IB_USER_VERBS_EX_CMD_QUERY_DEVICE)
+	    | BIT_ULL(IB_USER_VERBS_EX_CMD_CREATE_QP)
+	    | BIT_ULL(IB_USER_VERBS_EX_CMD_MODIFY_QP)
 	    | BIT_ULL(IB_USER_VERBS_EX_CMD_CREATE_CQ)
 	    | BIT_ULL(IB_USER_VERBS_EX_CMD_MODIFY_CQ)
 	    ;
Signature · 1 line
2.25.1
#15 lore raw

PATCH v7 13/19 rdma_rxe: Fix mcast group allocation bug

In reply to Bob Pearson

This patch does the following:
- Cleans up multicast group create to use an atomic sequence of lookup followed by allocate. This fixes an error that occurred when two QPs were attempting to attach to the same mcast address at the same time.
- Fixes a bug in rxe_mcast_get_grp not initializing err = 0. If the group is found in get_key the routine can return an uninitialized value to caller.
- Changes the variable elem to mce (for mcast elem). This is less likely to confuse readers because of the ambiguity with elem as pool element which is also used.

---
 drivers/infiniband/sw/rxe/rxe_mcast.c | 108 ++++++++++++++------------
 1 file changed, 60 insertions(+), 48 deletions(-)
drivers/infiniband/sw/rxe/rxe_mcast.c +60 -48
index b09c6594045a..541cc68a8a94 100644
@@ -7,44 +7,56 @@
 #include "rxe.h"
 #include "rxe_loc.h"
 
-int rxe_mcast_get_grp(struct rxe_dev *rxe, union ib_gid *mgid,
-		      struct rxe_mc_grp **grp_p)
+/* caller should hold mc_grp_pool->pool_lock */
+static struct rxe_mc_grp *create_grp(struct rxe_dev *rxe,
+					    struct rxe_pool *pool,
+					    union ib_gid *mgid)
 {
 	int err;
 	struct rxe_mc_grp *grp;
 
-	if (rxe->attr.max_mcast_qp_attach == 0) {
-		err = -EINVAL;
-		goto err1;
-	}
-
-	grp = rxe_get_key(&rxe->mc_grp_pool, mgid);
-	if (grp)
-		goto done;
-
-	grp = rxe_alloc(&rxe->mc_grp_pool);
-	if (!grp) {
-		err = -ENOMEM;
-		goto err1;
-	}
+	grp = __alloc(&rxe->mc_grp_pool);
+	if (unlikely(!grp))
+		return NULL;
 
 	INIT_LIST_HEAD(&grp->qp_list);
 	spin_lock_init(&grp->mcg_lock);
 	grp->rxe = rxe;
-
-	rxe_add_key(grp, mgid);
+	add_key(grp, mgid);
 
 	err = rxe_mcast_add(rxe, mgid);
-	if (err)
-		goto err2;
+	if (unlikely(err)) {
+		drop_key(grp);
+		rxe_drop_ref(grp);
+		return NULL;
+	}
 
+	return grp;
+}
+
+/* atomically lookup or create mc group */
+int rxe_mcast_get_grp(struct rxe_dev *rxe, union ib_gid *mgid,
+		      struct rxe_mc_grp **grp_p)
+{
+	int err = 0;
+	struct rxe_mc_grp *grp;
+	struct rxe_pool *pool = &rxe->mc_grp_pool;
+	unsigned long flags;
+
+	if (unlikely(rxe->attr.max_mcast_qp_attach == 0))
+		return -EINVAL;
+
+	write_lock_irqsave(&pool->pool_lock, flags);
+	grp = __get_key(pool, mgid);
+	if (grp)
+		goto done;
+
+	grp = create_grp(rxe, pool, mgid);
+	if (unlikely(!grp))
+		err = -ENOMEM;
 done:
+	write_unlock_irqrestore(&pool->pool_lock, flags);
 	*grp_p = grp;
-	return 0;
-
-err2:
-	rxe_drop_ref(grp);
-err1:
 	return err;
 }
 
@@ -52,13 +64,13 @@ int rxe_mcast_add_grp_elem(struct rxe_dev *rxe, struct rxe_qp *qp,
 			   struct rxe_mc_grp *grp)
 {
 	int err;
-	struct rxe_mc_elem *elem;
+	struct rxe_mc_elem *mce;
 
 	/* check to see of the qp is already a member of the group */
 	spin_lock_bh(&qp->grp_lock);
 	spin_lock_bh(&grp->mcg_lock);
-	list_for_each_entry(elem, &grp->qp_list, qp_list) {
-		if (elem->qp == qp) {
+	list_for_each_entry(mce, &grp->qp_list, qp_list) {
+		if (mce->qp == qp) {
 			err = 0;
 			goto out;
 		}
@@ -69,8 +81,8 @@ int rxe_mcast_add_grp_elem(struct rxe_dev *rxe, struct rxe_qp *qp,
 		goto out;
 	}
 
-	elem = rxe_alloc(&rxe->mc_elem_pool);
-	if (!elem) {
+	mce = rxe_alloc(&rxe->mc_elem_pool);
+	if (!mce) {
 		err = -ENOMEM;
 		goto out;
 	}
@@ -79,11 +91,11 @@ int rxe_mcast_add_grp_elem(struct rxe_dev *rxe, struct rxe_qp *qp,
 	rxe_add_ref(grp);
 
 	grp->num_qp++;
-	elem->qp = qp;
-	elem->grp = grp;
+	mce->qp = qp;
+	mce->grp = grp;
 
-	list_add(&elem->qp_list, &grp->qp_list);
-	list_add(&elem->grp_list, &qp->grp_list);
+	list_add(&mce->qp_list, &grp->qp_list);
+	list_add(&mce->grp_list, &qp->grp_list);
 
 	err = 0;
 out:
@@ -96,7 +108,7 @@ int rxe_mcast_drop_grp_elem(struct rxe_dev *rxe, struct rxe_qp *qp,
 			    union ib_gid *mgid)
 {
 	struct rxe_mc_grp *grp;
-	struct rxe_mc_elem *elem, *tmp;
+	struct rxe_mc_elem *mce, *tmp;
 
 	grp = rxe_get_key(&rxe->mc_grp_pool, mgid);
 	if (!grp)
@@ -105,15 +117,15 @@ int rxe_mcast_drop_grp_elem(struct rxe_dev *rxe, struct rxe_qp *qp,
 	spin_lock_bh(&qp->grp_lock);
 	spin_lock_bh(&grp->mcg_lock);
 
-	list_for_each_entry_safe(elem, tmp, &grp->qp_list, qp_list) {
-		if (elem->qp == qp) {
-			list_del(&elem->qp_list);
-			list_del(&elem->grp_list);
+	list_for_each_entry_safe(mce, tmp, &grp->qp_list, qp_list) {
+		if (mce->qp == qp) {
+			list_del(&mce->qp_list);
+			list_del(&mce->grp_list);
 			grp->num_qp--;
 
 			spin_unlock_bh(&grp->mcg_lock);
 			spin_unlock_bh(&qp->grp_lock);
-			rxe_drop_ref(elem);
+			rxe_drop_ref(mce);
 			rxe_drop_ref(grp);	/* ref held by QP */
 			rxe_drop_ref(grp);	/* ref from get_key */
 			return 0;
@@ -130,7 +142,7 @@ int rxe_mcast_drop_grp_elem(struct rxe_dev *rxe, struct rxe_qp *qp,
 void rxe_drop_all_mcast_groups(struct rxe_qp *qp)
 {
 	struct rxe_mc_grp *grp;
-	struct rxe_mc_elem *elem;
+	struct rxe_mc_elem *mce;
 
 	while (1) {
 		spin_lock_bh(&qp->grp_lock);
@@ -138,24 +150,24 @@ void rxe_drop_all_mcast_groups(struct rxe_qp *qp)
 			spin_unlock_bh(&qp->grp_lock);
 			break;
 		}
-		elem = list_first_entry(&qp->grp_list, struct rxe_mc_elem,
+		mce = list_first_entry(&qp->grp_list, struct rxe_mc_elem,
 					grp_list);
-		list_del(&elem->grp_list);
+		list_del(&mce->grp_list);
 		spin_unlock_bh(&qp->grp_lock);
 
-		grp = elem->grp;
+		grp = mce->grp;
 		spin_lock_bh(&grp->mcg_lock);
-		list_del(&elem->qp_list);
+		list_del(&mce->qp_list);
 		grp->num_qp--;
 		spin_unlock_bh(&grp->mcg_lock);
 		rxe_drop_ref(grp);
-		rxe_drop_ref(elem);
+		rxe_drop_ref(mce);
 	}
 }
 
-void rxe_mc_cleanup(struct rxe_pool_entry *arg)
+void rxe_mc_cleanup(struct rxe_pool_entry *pelem)
 {
-	struct rxe_mc_grp *grp = container_of(arg, typeof(*grp), pelem);
+	struct rxe_mc_grp *grp = container_of(pelem, struct rxe_mc_grp, pelem);
 	struct rxe_dev *rxe = grp->rxe;
 
 	rxe_drop_key(grp);
Signature · 1 line
2.25.1
#16 lore raw

PATCH v7 14/19 rdma_rxe: Fix bugs in the multicast receive path

In reply to Bob Pearson

This patch does the following
- Fix a bug in rxe_rcv.
The current code calls rxe_match_dgid which checks to see if the destination ip address (dgid) matches one of the addresses in the gid table. This is ok for unicast adfdresses but not mcast addresses. Because of this all mcast packets were previously dropped.
- Fix a bug in rxe_rcv_mcast_pkt.
The current code is just wrong. It assumed that it could pass the same skb to rxe_rcv_pkt changing the qp pointer as it went when multiple QPs were attached to the same mcast address. In fact each QP needs a separate clone of the skb which it will delete later.

---
 drivers/infiniband/sw/rxe/rxe_recv.c | 61 ++++++++++++++++------------
 1 file changed, 36 insertions(+), 25 deletions(-)
drivers/infiniband/sw/rxe/rxe_recv.c +36 -25
index e14cb20c4596..dbb623ec5eaa 100644
@@ -233,6 +233,8 @@ static void rxe_rcv_mcast_pkt(struct rxe_dev *rxe, struct sk_buff *skb)
 	struct rxe_mc_elem *mce;
 	struct rxe_qp *qp;
 	union ib_gid dgid;
+	struct sk_buff *per_qp_skb;
+	struct rxe_pkt_info *per_qp_pkt;
 	int err;
 
 	if (skb->protocol == htons(ETH_P_IP))
@@ -261,42 +263,36 @@ static void rxe_rcv_mcast_pkt(struct rxe_dev *rxe, struct sk_buff *skb)
 		if (err)
 			continue;
 
-		/* if *not* the last qp in the list
-		 * increase the users of the skb then post to the next qp
+		/* for all but the last qp create a new clone of the
+		 * skb and pass to the qp.
+		 * This effectively reverts an earlier change
+		 * which did not work. The pkt struct is contained
+		 * in the skb so each time you changed pkt you also
+		 * changed all the earlier pkts as well. Caused a mess.
 		 */
 		if (mce->qp_list.next != &mcg->qp_list)
-			skb_get(skb);
+			per_qp_skb = skb_clone(skb, GFP_ATOMIC);
+		else
+			per_qp_skb = skb;
 
-		pkt->qp = qp;
+		per_qp_pkt = SKB_TO_PKT(per_qp_skb);
+		per_qp_pkt->qp = qp;
 		rxe_add_ref(qp);
-		rxe_rcv_pkt(pkt, skb);
+		rxe_rcv_pkt(per_qp_pkt, per_qp_skb);
 	}
 
 	spin_unlock_bh(&mcg->mcg_lock);
-
-	rxe_drop_ref(mcg);	/* drop ref from rxe_get_key. */
+	rxe_drop_ref(mcg);	/* drop ref from rxe_pool_get_key. */
+	return;
 
 err1:
 	kfree_skb(skb);
 }
 
-static int rxe_match_dgid(struct rxe_dev *rxe, struct sk_buff *skb)
+static int rxe_match_dgid(struct rxe_dev *rxe, struct sk_buff *skb,
+			  union ib_gid *pdgid)
 {
-	struct rxe_pkt_info *pkt = SKB_TO_PKT(skb);
 	const struct ib_gid_attr *gid_attr;
-	union ib_gid dgid;
-	union ib_gid *pdgid;
-
-	if (pkt->mask & RXE_LOOPBACK_MASK)
-		return 0;
-
-	if (skb->protocol == htons(ETH_P_IP)) {
-		ipv6_addr_set_v4mapped(ip_hdr(skb)->daddr,
-				       (struct in6_addr *)&dgid);
-		pdgid = &dgid;
-	} else {
-		pdgid = (union ib_gid *)&ipv6_hdr(skb)->daddr;
-	}
 
 	gid_attr = rdma_find_gid_by_port(&rxe->ib_dev, pdgid,
 					 IB_GID_TYPE_ROCE_UDP_ENCAP,
@@ -314,17 +310,32 @@ void rxe_rcv(struct sk_buff *skb)
 	int err;
 	struct rxe_pkt_info *pkt = SKB_TO_PKT(skb);
 	struct rxe_dev *rxe = pkt->rxe;
+	union ib_gid dgid;
+	union ib_gid *pdgid;
 	__be32 *icrcp;
 	u32 calc_icrc, pack_icrc;
+	int is_mc;
 
 	pkt->offset = 0;
 
 	if (unlikely(skb->len < pkt->offset + RXE_BTH_BYTES))
 		goto drop;
 
-	if (rxe_match_dgid(rxe, skb) < 0) {
-		pr_warn_ratelimited("failed matching dgid\n");
-		goto drop;
+	if (skb->protocol == htons(ETH_P_IP)) {
+		ipv6_addr_set_v4mapped(ip_hdr(skb)->daddr,
+				       (struct in6_addr *)&dgid);
+		pdgid = &dgid;
+	} else {
+		pdgid = (union ib_gid *)&ipv6_hdr(skb)->daddr;
+	}
+
+	is_mc = rdma_is_multicast_addr((struct in6_addr *)pdgid);
+
+	if (!(pkt->mask & RXE_LOOPBACK_MASK) && !is_mc) {
+		if (rxe_match_dgid(rxe, skb, pdgid) < 0) {
+			pr_warn_ratelimited("failed matching dgid\n");
+			goto drop;
+		}
 	}
 
 	pkt->opcode = bth_opcode(pkt);
Signature · 1 line
2.25.1
#17 lore raw

PATCH v7 15/19 rdma_rxe: handle ERR_PTR returns from pool

In reply to Bob Pearson

v7:
- rxe_pool.c has been extended to return errors from alloc, get_index and get_key using ERR_PTR. Add code to detect these as needed.

---
 drivers/infiniband/sw/rxe/rxe_mcast.c | 28 ++++++++++++++++++---------
 drivers/infiniband/sw/rxe/rxe_mr.c    |  2 +-
 drivers/infiniband/sw/rxe/rxe_mw.c    | 11 +++++++++++
 drivers/infiniband/sw/rxe/rxe_pool.c  |  6 +++---
 drivers/infiniband/sw/rxe/rxe_recv.c  |  4 +++-
 drivers/infiniband/sw/rxe/rxe_req.c   |  4 ++--
 drivers/infiniband/sw/rxe/rxe_resp.c  |  8 ++++----
 drivers/infiniband/sw/rxe/rxe_verbs.c | 16 +++++++--------
 8 files changed, 51 insertions(+), 28 deletions(-)
drivers/infiniband/sw/rxe/rxe_mcast.c +19 -9
index 541cc68a8a94..0dd8abefc13f 100644
@@ -16,8 +16,8 @@ static struct rxe_mc_grp *create_grp(struct rxe_dev *rxe,
 	struct rxe_mc_grp *grp;
 
 	grp = __alloc(&rxe->mc_grp_pool);
-	if (unlikely(!grp))
-		return NULL;
+	if (IS_ERR(grp))
+		return grp;
 
 	INIT_LIST_HEAD(&grp->qp_list);
 	spin_lock_init(&grp->mcg_lock);
@@ -28,7 +28,7 @@ static struct rxe_mc_grp *create_grp(struct rxe_dev *rxe,
 	if (unlikely(err)) {
 		drop_key(grp);
 		rxe_drop_ref(grp);
-		return NULL;
+		return ERR_PTR(err);
 	}
 
 	return grp;
@@ -43,20 +43,30 @@ int rxe_mcast_get_grp(struct rxe_dev *rxe, union ib_gid *mgid,
 	struct rxe_pool *pool = &rxe->mc_grp_pool;
 	unsigned long flags;
 
-	if (unlikely(rxe->attr.max_mcast_qp_attach == 0))
-		return -EINVAL;
+	if (unlikely(rxe->attr.max_mcast_qp_attach == 0)) {
+		err = -EINVAL;
+		goto err;
+	}
 
 	write_lock_irqsave(&pool->pool_lock, flags);
 	grp = __get_key(pool, mgid);
+	if (IS_ERR(grp))
+		goto err_ptr;
 	if (grp)
 		goto done;
 
 	grp = create_grp(rxe, pool, mgid);
-	if (unlikely(!grp))
-		err = -ENOMEM;
+	if (IS_ERR(grp))
+		goto err_ptr;
 done:
 	write_unlock_irqrestore(&pool->pool_lock, flags);
 	*grp_p = grp;
+	return 0;
+
+err_ptr:
+	write_unlock_irqrestore(&pool->pool_lock, flags);
+	err = PTR_ERR(grp);
+err:
 	return err;
 }
 
@@ -82,8 +92,8 @@ int rxe_mcast_add_grp_elem(struct rxe_dev *rxe, struct rxe_qp *qp,
 	}
 
 	mce = rxe_alloc(&rxe->mc_elem_pool);
-	if (!mce) {
-		err = -ENOMEM;
+	if (IS_ERR(mce)) {
+		err = PTR_ERR(mce);
 		goto out;
 	}
 
drivers/infiniband/sw/rxe/rxe_mr.c +1 -1
index c3ad89d85360..e3eb0f4bb2a0 100644
@@ -386,7 +386,7 @@ static struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 lkey)
 	struct rxe_dev *rxe = to_rdev(pd->ibpd.device);
 
 	mr = rxe_get_key(&rxe->mr_pool, &lkey);
-	if (!mr)
+	if (IS_ERR(mr) || !mr)
 		return NULL;
 
 	if (unlikely((mr->ibmr.lkey != lkey) || (mr->pd != pd) ||
drivers/infiniband/sw/rxe/rxe_mw.c +11 -0
index 22d3570ac184..d2d09502a28d 100644
@@ -217,6 +217,8 @@ static int do_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
 	rkey = mw->ibmw.rkey;
 	new_rkey = (rkey & 0xffffff00) | (wqe->wr.wr.umw.rkey & 0x000000ff);
 	duplicate_mw = rxe_get_key(&rxe->mw_pool, &new_rkey);
+	if (IS_ERR(duplicate_mw))
+		return PTR_ERR(duplicate_mw);
 	if (duplicate_mw) {
 		pr_err_once("new MW key is a duplicate, try another\n");
 		rxe_drop_ref(duplicate_mw);
@@ -260,14 +262,23 @@ int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
 	if (qp->is_user) {
 		mw = rxe_get_index(&rxe->mw_pool,
 				   wqe->wr.wr.umw.mw_index);
+		if (IS_ERR(mw)) {
+			ret = PTR_ERR(mw);
+			goto err1;
+		}
 		if (!mw) {
 			pr_err_once("mw with index = %d not found\n",
 			wqe->wr.wr.umw.mw_index);
 			ret = -EINVAL;
 			goto err1;
 		}
+
 		mr = rxe_get_index(&rxe->mr_pool,
 				   wqe->wr.wr.umw.mr_index);
+		if (IS_ERR(mr)) {
+			ret = PTR_ERR(mr);
+			goto err2;
+		}
 		if (!mr && wqe->wr.wr.umw.length) {
 			pr_err_once("mr with index = %d not found\n",
 			wqe->wr.wr.umw.mr_index);
drivers/infiniband/sw/rxe/rxe_pool.c +3 -3
index 9d53b4d71230..76e6609dcf02 100644
@@ -581,7 +581,7 @@ void *rxe_alloc(struct rxe_pool *pool)
 			(pool->flags & RXE_POOL_ATOMIC) ?
 			GFP_ATOMIC : GFP_KERNEL);
 	if (!obj)
-		goto out;
+		return ERR_PTR(-ENOMEM);
 
 	elem = (struct rxe_pool_entry *)((u8 *)obj +
 			rxe_type_info[pool->type].elem_offset);
@@ -591,9 +591,9 @@ void *rxe_alloc(struct rxe_pool *pool)
 	write_unlock_irqrestore(&pool->pool_lock, flags);
 	if (err) {
 		kfree(obj);
-		obj = NULL;
+		return ERR_PTR(err);
 	}
-out:
+
 	return obj;
 }
 
drivers/infiniband/sw/rxe/rxe_recv.c +3 -1
index dbb623ec5eaa..213c979b6b46 100644
@@ -186,6 +186,8 @@ static int hdr_check(struct rxe_pkt_info *pkt)
 		index = (qpn == 1) ? port->qp_gsi_index : qpn;
 
 		qp = rxe_get_index(&rxe->qp_pool, index);
+		if (IS_ERR(qp))
+			goto err1;
 		if (unlikely(!qp)) {
 			pr_warn_ratelimited("no qp matches qpn 0x%x\n", qpn);
 			goto err1;
@@ -245,7 +247,7 @@ static void rxe_rcv_mcast_pkt(struct rxe_dev *rxe, struct sk_buff *skb)
 
 	/* lookup mcast group corresponding to mgid, takes a ref */
 	mcg = rxe_get_key(&rxe->mc_grp_pool, &dgid);
-	if (!mcg)
+	if (IS_ERR(mcg) || !mcg)
 		goto err1;	/* mcast group not registered */
 
 	spin_lock_bh(&mcg->mcg_lock);
drivers/infiniband/sw/rxe/rxe_req.c +2 -2
index 0428965c664b..3e815158df55 100644
@@ -565,7 +565,7 @@ static int invalidate_key(struct rxe_qp *qp, u32 key)
 
 	if (key & IS_MW) {
 		mw = rxe_get_key(&rxe->mw_pool, &key);
-		if (!mw) {
+		if (IS_ERR(mw) || !mw) {
 			pr_err("No mw for key %#x\n", key);
 			return -EINVAL;
 		}
@@ -573,7 +573,7 @@ static int invalidate_key(struct rxe_qp *qp, u32 key)
 		rxe_drop_ref(mw);
 	} else {
 		mr = rxe_get_key(&rxe->mr_pool, &key);
-		if (!mr) {
+		if (IS_ERR(mr) || !mr) {
 			pr_err("No mr for key %#x\n", key);
 			return -EINVAL;
 		}
drivers/infiniband/sw/rxe/rxe_resp.c +4 -4
index 58d3783063bb..47d863c4dd48 100644
@@ -442,7 +442,7 @@ static enum resp_states check_rkey(struct rxe_qp *qp,
 	 */
 	if (rkey & IS_MW) {
 		mw = rxe_get_key(&rxe->mw_pool, &rkey);
-		if (!mw) {
+		if (IS_ERR(mw) || !mw) {
 			pr_err_once("no MW found with rkey = 0x%08x\n", rkey);
 			state = RESPST_ERR_RKEY_VIOLATION;
 			goto err;
@@ -466,7 +466,7 @@ static enum resp_states check_rkey(struct rxe_qp *qp,
 		rxe_drop_ref(mw);
 	} else {
 		mr = rxe_get_key(&rxe->mr_pool, &rkey);
-		if (!mr || (mr->rkey != rkey)) {
+		if (IS_ERR(mr) || !mr || (mr->rkey != rkey)) {
 			pr_err_once("no MR found with rkey = 0x%08x\n", rkey);
 			state = RESPST_ERR_RKEY_VIOLATION;
 			goto err;
@@ -794,7 +794,7 @@ static int invalidate_rkey(struct rxe_qp *qp, u32 rkey)
 
 	if (rkey & IS_MW) {
 		mw = rxe_get_key(&rxe->mw_pool, &rkey);
-		if (!mw) {
+		if (IS_ERR(mw) || !mw) {
 			pr_err("No mw for rkey %#x\n", rkey);
 			goto err;
 		}
@@ -802,7 +802,7 @@ static int invalidate_rkey(struct rxe_qp *qp, u32 rkey)
 		rxe_drop_ref(mw);
 	} else {
 		mr = rxe_get_key(&rxe->mr_pool, &rkey);
-		if (!mr || mr->ibmr.rkey != rkey) {
+		if (IS_ERR(mr) || !mr || mr->ibmr.rkey != rkey) {
 			pr_err("No mr for rkey %#x\n", rkey);
 			goto err;
 		}
drivers/infiniband/sw/rxe/rxe_verbs.c +8 -8
index 4de263e41b07..58a3b66e6283 100644
@@ -397,8 +397,8 @@ static struct ib_qp *rxe_create_qp(struct ib_pd *ibpd,
 		goto err1;
 
 	qp = rxe_alloc(&rxe->qp_pool);
-	if (!qp) {
-		err = -ENOMEM;
+	if (IS_ERR(qp)) {
+		err = PTR_ERR(qp);
 		goto err1;
 	}
 
@@ -872,9 +872,9 @@ static struct ib_mr *rxe_get_dma_mr(struct ib_pd *ibpd, int access)
 	rxe_add_ref(pd);
 
 	mr = rxe_alloc(&rxe->mr_pool);
-	if (!mr) {
+	if (IS_ERR(mr)) {
 		rxe_drop_ref(pd);
-		return ERR_PTR(-ENOMEM);
+		return (void *)mr;
 	}
 
 	rxe_mr_init_dma(pd, access, mr);
@@ -905,8 +905,8 @@ static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd,
 	rxe_add_ref(pd);
 
 	mr = rxe_alloc(&rxe->mr_pool);
-	if (!mr) {
-		err = -ENOMEM;
+	if (IS_ERR(mr)) {
+		err = PTR_ERR(mr);
 		goto err2;
 	}
 
@@ -956,8 +956,8 @@ static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
 	rxe_add_ref(pd);
 
 	mr = rxe_alloc(&rxe->mr_pool);
-	if (!mr) {
-		err = -ENOMEM;
+	if (IS_ERR(mr)) {
+		err = PTR_ERR(mr);
 		goto err1;
 	}
 
Signature · 1 line
2.25.1
#18 lore raw

PATCH v7 16/19 rdma_rxe: remove duplicate entries in struct rxe_mr

In reply to Bob Pearson

v7:
- Struct rxe_mr had lkey and rkey values both in itself and in the struct ib_mr. These are deleted and references replaced to the ones in ibmr.
- Removed the struct rxe_pd pointer from struct rxe_mr. This duplicates the pd pointer in struct ib_mr which was also present. Added a mr_pd() macro which extracts the pd pointer from mr.

---
 drivers/infiniband/sw/rxe/rxe_mr.c    | 11 +++--------
 drivers/infiniband/sw/rxe/rxe_req.c   |  4 ++--
 drivers/infiniband/sw/rxe/rxe_resp.c  |  2 +-
 drivers/infiniband/sw/rxe/rxe_verbs.c |  2 +-
 drivers/infiniband/sw/rxe/rxe_verbs.h |  8 ++++----
 5 files changed, 11 insertions(+), 16 deletions(-)
drivers/infiniband/sw/rxe/rxe_mr.c +3 -8
index e3eb0f4bb2a0..f96f908644b1 100644
@@ -51,8 +51,6 @@ static void rxe_mr_init(int access, struct rxe_mr *mr)
 	if (access & IB_ACCESS_REMOTE)
 		mr->ibmr.rkey = mr->ibmr.lkey;
 
-	mr->lkey		= mr->ibmr.lkey;
-	mr->rkey		= mr->ibmr.rkey;
 	mr->state		= RXE_MEM_STATE_INVALID;
 	mr->type		= RXE_MR_TYPE_NONE;
 	mr->map_shift		= ilog2(RXE_BUF_PER_MAP);
@@ -101,7 +99,6 @@ void rxe_mr_init_dma(struct rxe_pd *pd,
 {
 	rxe_mr_init(access, mr);
 
-	mr->pd			= pd;
 	mr->access		= access;
 	mr->state		= RXE_MEM_STATE_VALID;
 	mr->type		= RXE_MR_TYPE_DMA;
@@ -170,7 +167,6 @@ int rxe_mr_init_user(struct rxe_pd *pd, u64 start,
 		}
 	}
 
-	mr->pd			= pd;
 	mr->umem		= umem;
 	mr->access		= access;
 	mr->length		= length;
@@ -200,7 +196,6 @@ int rxe_mr_init_fast(struct rxe_pd *pd,
 	if (err)
 		goto err1;
 
-	mr->pd			= pd;
 	mr->max_buf		= max_pages;
 	mr->state		= RXE_MEM_STATE_FREE;
 	mr->type		= RXE_MR_TYPE_MR;
@@ -320,7 +315,7 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
 		memcpy(dest, src, length);
 
 		if (crcp)
-			*crcp = rxe_crc32(to_rdev(mr->pd->ibpd.device),
+			*crcp = rxe_crc32(to_rdev(mr->ibmr.device),
 					*crcp, dest, length);
 
 		return 0;
@@ -354,7 +349,7 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
 		memcpy(dest, src, bytes);
 
 		if (crcp)
-			crc = rxe_crc32(to_rdev(mr->pd->ibpd.device),
+			crc = rxe_crc32(to_rdev(mr->ibmr.device),
 					crc, dest, bytes);
 
 		length	-= bytes;
@@ -389,7 +384,7 @@ static struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 lkey)
 	if (IS_ERR(mr) || !mr)
 		return NULL;
 
-	if (unlikely((mr->ibmr.lkey != lkey) || (mr->pd != pd) ||
+	if (unlikely((mr_lkey(mr) != lkey) || (mr_pd(mr) != pd) ||
 		     (access && !(access & mr->access)) ||
 		     (mr->state != RXE_MEM_STATE_VALID))) {
 		rxe_drop_ref(mr);
drivers/infiniband/sw/rxe/rxe_req.c +2 -2
index 3e815158df55..128ec5d79501 100644
@@ -638,8 +638,8 @@ int rxe_requester(void *arg)
 			mr = to_rmr(wqe->wr.wr.reg.mr);
 			mr->state = RXE_MEM_STATE_VALID;
 			mr->access = wqe->wr.wr.reg.access;
-			mr->lkey = wqe->wr.wr.reg.key;
-			mr->rkey = wqe->wr.wr.reg.key;
+			mr->ibmr.lkey = wqe->wr.wr.reg.key;
+			mr->ibmr.rkey = wqe->wr.wr.reg.key;
 			mr->iova = wqe->wr.wr.reg.mr->iova;
 			break;
 		case IB_WR_BIND_MW:
drivers/infiniband/sw/rxe/rxe_resp.c +1 -1
index 47d863c4dd48..b8eb4f43c312 100644
@@ -466,7 +466,7 @@ static enum resp_states check_rkey(struct rxe_qp *qp,
 		rxe_drop_ref(mw);
 	} else {
 		mr = rxe_get_key(&rxe->mr_pool, &rkey);
-		if (IS_ERR(mr) || !mr || (mr->rkey != rkey)) {
+		if (IS_ERR(mr) || !mr || (mr_rkey(mr) != rkey)) {
 			pr_err_once("no MR found with rkey = 0x%08x\n", rkey);
 			state = RESPST_ERR_RKEY_VIOLATION;
 			goto err;
drivers/infiniband/sw/rxe/rxe_verbs.c +1 -1
index 58a3b66e6283..fc00231627e7 100644
@@ -937,7 +937,7 @@ static int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
 	struct rxe_mr *mr = to_rmr(ibmr);
 
 	mr->state = RXE_MEM_STATE_ZOMBIE;
-	rxe_drop_ref(mr->pd);
+	rxe_drop_ref(mr_pd(mr));
 	rxe_drop_ref(mr);
 	return 0;
 }
drivers/infiniband/sw/rxe/rxe_verbs.h +4 -4
index 54429d8a06fb..847f80d7a1b6 100644
@@ -293,12 +293,8 @@ struct rxe_mr {
 	struct rxe_pool_entry	pelem;
 	struct ib_mr		ibmr;
 
-	struct rxe_pd		*pd;
 	struct ib_umem		*umem;
 
-	u32			lkey;
-	u32			rkey;
-
 	enum rxe_mem_state	state;
 	enum rxe_mr_type	type;
 	u64			va;
@@ -323,6 +319,10 @@ struct rxe_mr {
 	struct rxe_map		**map;
 };
 
+#define mr_pd(mr) to_rpd((mr)->ibmr.pd)
+#define mr_lkey(mr) ((mr)->ibmr.lkey)
+#define mr_rkey(mr) ((mr)->ibmr.rkey)
+
 enum rxe_send_flags {
 	/* flag indicaes bind call came through verbs API */
 	RXE_BIND_MW		= (1 << 0),
Signature · 1 line
2.25.1
#19 lore raw

PATCH v7 17/19 rdma_rme: removed unused RXE_MR_TYPE_FMR

In reply to Bob Pearson

This is a left over from the past. A step in replacing rxe mr types with ib mr types.

---
 drivers/infiniband/sw/rxe/rxe_mr.c    | 1 -
 drivers/infiniband/sw/rxe/rxe_verbs.h | 1 -
 2 files changed, 2 deletions(-)
drivers/infiniband/sw/rxe/rxe_mr.c +0 -1
index f96f908644b1..1901f388c747 100644
@@ -28,7 +28,6 @@ static int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)
 		return 0;
 
 	case RXE_MR_TYPE_MR:
-	case RXE_MR_TYPE_FMR:
 		if (iova < mr->iova ||
 		    length > mr->length ||
 		    iova > mr->iova + mr->length - length)
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -1
index 847f80d7a1b6..ea9861998413 100644
@@ -275,7 +275,6 @@ enum rxe_mr_type {
 	RXE_MR_TYPE_NONE,
 	RXE_MR_TYPE_DMA,
 	RXE_MR_TYPE_MR,
-	RXE_MR_TYPE_FMR,
 };
 
 #define RXE_BUF_PER_MAP		(PAGE_SIZE / sizeof(struct rxe_phys_buf))
Signature · 1 line
2.25.1
#20 lore raw

PATCH v7 18/19 rdma_rxe: add rkey validation checks for MR and MW

In reply to Bob Pearson

Added checks from 9.4.1.1.3 r_key validation for remote memory invalidate to MR and MW invalidate routines.

---
 drivers/infiniband/sw/rxe/rxe_mr.c | 49 ++++++++++++++++++++++++++++++
 drivers/infiniband/sw/rxe/rxe_mw.c | 10 +++++-
 2 files changed, 58 insertions(+), 1 deletion(-)
drivers/infiniband/sw/rxe/rxe_mr.c +49 -0
index 1901f388c747..2cc3487d3f77 100644
@@ -522,8 +522,57 @@ int advance_dma_data(struct rxe_dma_info *dma, unsigned int length)
 	return 0;
 }
 
+static int check_invalidate_mr(struct rxe_qp *qp, struct rxe_mr *mr)
+{
+	int type = mr->ibmr.type;
+	int qpt = qp_type(qp);
+
+	if (unlikely(type > IB_MR_TYPE_INTEGRITY)) {
+		pr_err("invalidate: MR of unknown type = %d\n",
+			mr->ibmr.type);
+		return -EINVAL;
+	}
+
+	/* o10-37.2.13 */
+	if (unlikely(type == IB_MR_TYPE_MEM_REG ||
+	    type == IB_MR_TYPE_USER)) {
+		pr_err("invalidate: MR created by reg_mr or user_reg_mr\n");
+		return -EINVAL;
+	}
+
+	/* o10-37.2.17 */
+	if (unlikely(atomic_read(&mr->num_mw))) {
+		pr_err("invalidate: MR with bound MWs\n");
+		return -EINVAL;
+	}
+
+	if (unlikely(!((qpt == IB_QPT_RC) || (qpt == IB_QPT_UC) ||
+		(qpt == IB_QPT_XRC_INI) || (qpt == IB_QPT_XRC_TGT)))) {
+		pr_err("invalidate: MR with invalid QP type\n");
+		return -EINVAL;
+	}
+
+	if (unlikely(qp->ibqp.pd != mr->ibmr.pd)) {
+		pr_err("invalidate: MR and QP have different PDs\n");
+		return -EINVAL;
+	}
+
+	if (unlikely(mr->state == RXE_MEM_STATE_INVALID)) {
+		pr_err("invalidate: MR in invalid state\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 int rxe_invalidate_mr(struct rxe_qp *qp, struct rxe_mr *mr)
 {
+	int ret;
+
+	ret = check_invalidate_mr(qp, mr);
+	if (ret)
+		return ret;
+
 	/* TODO there are API rules being ignored here
 	 * cleanup later. Current project is not trying
 	 * to fix MR
drivers/infiniband/sw/rxe/rxe_mw.c +9 -1
index d2d09502a28d..ecda634ab7f0 100644
@@ -316,9 +316,17 @@ int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
 
 static int check_invalidate_mw(struct rxe_qp *qp, struct rxe_mw *mw)
 {
+	int qpt = qp_type(qp);
+
 	/* o10-37.2.26 */
 	if (unlikely(mw->ibmw.type == IB_MW_TYPE_1)) {
-		pr_err_once("attempt to invalidate a type 1 MW\n");
+		pr_err_once("invalidate a type 1 MW\n");
+		return -EINVAL;
+	}
+
+	if (unlikely(!((qpt == IB_QPT_RC) || (qpt == IB_QPT_UC) ||
+		(qpt == IB_QPT_XRC_INI) || (qpt == IB_QPT_XRC_TGT)))) {
+		pr_err("invalidate Mw with invalid QP type\n");
 		return -EINVAL;
 	}
 
Signature · 1 line
2.25.1
#21 lore raw

PATCH v7 19/19 rdma_rxe: moved rxe_xmit_packet to rxe_net.c

In reply to Bob Pearson

No good reason to stick such a large subroutine in a header file as an inline function.

---
 drivers/infiniband/sw/rxe/rxe_loc.h | 47 ++---------------------------
 drivers/infiniband/sw/rxe/rxe_net.c | 47 +++++++++++++++++++++++++++--
 2 files changed, 47 insertions(+), 47 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +2 -45
index fcffd5075b18..9d3c59694e7b 100644
@@ -124,8 +124,6 @@ int rxe_mw_check_access(struct rxe_qp *qp, struct rxe_mw *mw,
 void rxe_mw_cleanup(struct rxe_pool_entry *arg);
 
 /* rxe_net.c */
-void rxe_loopback(struct sk_buff *skb);
-int rxe_send(struct rxe_pkt_info *pkt, struct sk_buff *skb);
 struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av,
 				int paylen, struct rxe_pkt_info *pkt);
 int rxe_prepare(struct rxe_pkt_info *pkt, struct sk_buff *skb, u32 *crc);
@@ -133,6 +131,8 @@ const char *rxe_parent_name(struct rxe_dev *rxe, unsigned int port_num);
 struct device *rxe_dma_device(struct rxe_dev *rxe);
 int rxe_mcast_add(struct rxe_dev *rxe, union ib_gid *mgid);
 int rxe_mcast_delete(struct rxe_dev *rxe, union ib_gid *mgid);
+int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
+		    struct sk_buff *skb);
 
 /* rxe_qp.c */
 int rxe_qp_chk_init(struct rxe_dev *rxe, struct ib_qp_init_attr *init);
@@ -230,47 +230,4 @@ static inline unsigned int wr_opcode_mask(int opcode, struct rxe_qp *qp)
 	return rxe_wr_opcode_info[opcode].mask[qp->ibqp.qp_type];
 }
 
-static inline int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
-				  struct sk_buff *skb)
-{
-	int err;
-	int is_request = pkt->mask & RXE_REQ_MASK;
-	struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
-
-	if ((is_request && (qp->req.state != QP_STATE_READY)) ||
-	    (!is_request && (qp->resp.state != QP_STATE_READY))) {
-		pr_info("Packet dropped. QP is not in ready state\n");
-		goto drop;
-	}
-
-	if (pkt->mask & RXE_LOOPBACK_MASK) {
-		memcpy(SKB_TO_PKT(skb), pkt, sizeof(*pkt));
-		rxe_loopback(skb);
-		err = 0;
-	} else {
-		err = rxe_send(pkt, skb);
-	}
-
-	if (err) {
-		rxe->xmit_errors++;
-		rxe_counter_inc(rxe, RXE_CNT_SEND_ERR);
-		return err;
-	}
-
-	if ((qp_type(qp) != IB_QPT_RC) &&
-	    (pkt->mask & RXE_END_MASK)) {
-		pkt->wqe->state = wqe_state_done;
-		rxe_run_task(&qp->comp.task, 1);
-	}
-
-	rxe_counter_inc(rxe, RXE_CNT_SENT_PKTS);
-	goto done;
-
-drop:
-	kfree_skb(skb);
-	err = 0;
-done:
-	return err;
-}
-
 #endif /* RXE_LOC_H */
drivers/infiniband/sw/rxe/rxe_net.c +45 -2

63 lines not on this page · raw message

Signature · 1 line
2.25.1