rdma_rxe: API extensions
Subsystems (2)
v7 cover rdma_rxe: API extensions
V7:
Ported this patch series forward to the current state of for-next.
MW now being allocated in core.
Split up rxe_pool changes into patches as requested by Jason. Added some better comments.
Added error returns via ERR_PTR for APIs that return pointers.
Added some additional validation for rkeys in rxe_resp based on chapter 9 of IBA.
Minor cleanup to rxe_mr which had duplicate entries.
V6:
Fixed two issues raised by Jason and Zhu.
Undid the replacement of rwlocks by spinlocks in patch 10/12. On further reading it turns out rwlocks were the better choice.
Missing prototype for rxe_invalidate_mr. This was caused by a regression in patch 05/12 which had dropped the actual use of the routine as well as the prototype. Fixed.
V5:
This patch series is a collection of API extensions for the rdma_rxe driver. With this patch set installed there are no errors in pyverbs run-tests and 31 tests are skipped down from 56. The remaining skipped test cases include
- XRC tests
- ODP tests
- Parent device tests
- Import tests
- Device memory
- MLX5 specific tests
- EFA tests
It continues from the previous (v4) set which implemented memory windows and has had a number of individual patches picked up in for-next.
This set (v5) includes:
Ported to current head of tree
Memory windows patches not yet picked up kernel support for the extended user space APIs:
- ibv_query_device_ex
- ibv_create_cq_ex
- ibv_create_qp_ex
Fixes for multicast which is not currently working
This patch set depends on a matching rdma-core user space library patch set.
In order to run correctly it is necessary to configure by hand the EUI64 link local IPV6 address on systems which use a random link local address (like Ubuntu).
Bob Pearson (19):
rdma_rxe: Separat MEM into MR and MW objects.
rdma_rxe: Enable MW objects
rdma_rxe: Let pools support both keys and indices
rdma_rxe: make pool return values position independent
rdma_rxe: remove void * parameters in pool APIs
rdma_rxe: add alloc_mw and dealloc_mw verbs
rdma_rxe: Add bind_mw and invalidate_mw verbs
rdma_rxe: Add memory access through MWs
rdma_rxe: Add locked and unlocked pool APIs
rdma_rxe: Add support for ibv_query_device_ex
rdma_rxe: Add support for extended CQ operations
rdma_rxe: Add support for extended QP operations
rdma_rxe: Fix mcast group allocation bug
rdma_rxe: Fix bugs in the multicast receive path
rdma_rxe: handle ERR_PTR returns from pool
rdma_rxe: remove duplicate entries in struct rxe_mr
rdma_rme: removed unused RXE_MR_TYPE_FMR
rdma_rxe: add rkey validation checks for MR and MW
rdma_rxe: moved rxe_xmit_packet to rxe_net.c
drivers/infiniband/sw/rxe/Makefile | 1 + drivers/infiniband/sw/rxe/rxe.c | 100 ++-- drivers/infiniband/sw/rxe/rxe_comp.c | 12 +- drivers/infiniband/sw/rxe/rxe_loc.h | 91 ++-- drivers/infiniband/sw/rxe/rxe_mcast.c | 118 +++-- drivers/infiniband/sw/rxe/rxe_mr.c | 396 ++++++++------- drivers/infiniband/sw/rxe/rxe_mw.c | 420 ++++++++++++++++ drivers/infiniband/sw/rxe/rxe_net.c | 47 +- drivers/infiniband/sw/rxe/rxe_opcode.c | 11 +- drivers/infiniband/sw/rxe/rxe_opcode.h | 1 - drivers/infiniband/sw/rxe/rxe_param.h | 10 +- drivers/infiniband/sw/rxe/rxe_pool.c | 653 ++++++++++++++++++------- drivers/infiniband/sw/rxe/rxe_pool.h | 148 +++--- drivers/infiniband/sw/rxe/rxe_recv.c | 67 ++- drivers/infiniband/sw/rxe/rxe_req.c | 145 ++++-- drivers/infiniband/sw/rxe/rxe_resp.c | 188 +++++-- drivers/infiniband/sw/rxe/rxe_verbs.c | 118 +++-- drivers/infiniband/sw/rxe/rxe_verbs.h | 71 ++- include/uapi/rdma/rdma_user_rxe.h | 68 ++- 19 files changed, 1930 insertions(+), 735 deletions(-) create mode 100644 drivers/infiniband/sw/rxe/rxe_mw.c
Signature · 1 line
2.25.1
PATCH v7 1/19 rdma_rxe: Separat MEM into MR and MW objects.
In the original rxe implementation it was intended to use a common object to represent MRs and MWs but it became clear that they are different enough to separate these into two objects.
This allows replacing the mem name with mr for MRs which is more consistent with the style for the other objects and less likely to be confusing. This is a long patch that mostly changes mem to mr where it makes sense and adds a new rxe_mw struct.
--- drivers/infiniband/sw/rxe/rxe_comp.c | 4 +- drivers/infiniband/sw/rxe/rxe_loc.h | 26 +-- drivers/infiniband/sw/rxe/rxe_mr.c | 264 +++++++++++++------------- drivers/infiniband/sw/rxe/rxe_pool.c | 8 +- drivers/infiniband/sw/rxe/rxe_req.c | 6 +- drivers/infiniband/sw/rxe/rxe_resp.c | 30 +-- drivers/infiniband/sw/rxe/rxe_verbs.c | 18 +- drivers/infiniband/sw/rxe/rxe_verbs.h | 51 ++--- 8 files changed, 204 insertions(+), 203 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +2 -2
@@ -345,7 +345,7 @@ static inline enum comp_state do_read(struct rxe_qp *qp, ret =copy_data (qp->pd,IB_ACCESS_LOCAL_WRITE , &wqe->dma,payload_addr (pkt), -payload_size (pkt), to_mem_obj,NULL ); +payload_size (pkt), to_mr_obj,NULL );if (ret)return COMPST_ERROR ;
@@ -365,7 +365,7 @@ static inline enum comp_state do_atomic(struct rxe_qp *qp, ret =copy_data (qp->pd,IB_ACCESS_LOCAL_WRITE , &wqe->dma, &atomic_orig, -sizeof (u64), to_mem_obj,NULL ); +sizeof (u64), to_mr_obj,NULL );if (ret)return COMPST_ERROR ;else
drivers/infiniband/sw/rxe/rxe_loc.h +13 -13
@@ -72,40 +72,40 @@ int rxe_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);/* rxe_mr.c */ enum copy_direction { - to_mem_obj, - from_mem_obj, + to_mr_obj, + from_mr_obj, }; -void rxe_mem_init_dma (struct rxe_pd *pd, -int access,struct rxe_mem *mem); +void rxe_mr_init_dma (struct rxe_pd *pd, +int access,struct rxe_mr *mr); -int rxe_mem_init_user (struct rxe_pd *pd,u64 start, +int rxe_mr_init_user (struct rxe_pd *pd,u64 start,u64 length,u64 iova,int access,struct ib_udata *udata, -struct rxe_mem *mr); +struct rxe_mr *mr); -int rxe_mem_init_fast (struct rxe_pd *pd, -int max_pages,struct rxe_mem *mem); +int rxe_mr_init_fast (struct rxe_pd *pd, +int max_pages,struct rxe_mr *mr); -int rxe_mem_copy (struct rxe_mem *mem,u64 iova,void *addr, +int rxe_mr_copy (struct rxe_mr *mr,u64 iova,void *addr,int length,enum copy_direction dir,u32 *crcp);int copy_data (struct rxe_pd *pd,int access,struct rxe_dma_info *dma,void *addr,int length,enum copy_direction dir,u32 *crcp); -void *iova_to_vaddr (struct rxe_mem *mem,u64 iova,int length); +void *iova_to_vaddr (struct rxe_mr *mr,u64 iova,int length);enum lookup_type { lookup_local, lookup_remote, }; -struct rxe_mem *lookup_mem (struct rxe_pd *pd,int access,u32 key, +struct rxe_mr *lookup_mr (struct rxe_pd *pd,int access,u32 key,enum lookup_type type); -int mem_check_range (struct rxe_mem *mem,u64 iova,size_t length); +int mr_check_range (struct rxe_mr *mr,u64 iova,size_t length); -void rxe_mem_cleanup (struct rxe_pool_entry *arg); +void rxe_mr_cleanup (struct rxe_pool_entry *arg);int advance_dma_data (struct rxe_dma_info *dma,unsigned int length);
drivers/infiniband/sw/rxe/rxe_mr.c +132 -132
518 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_pool.c +3 -5
@@ -8,8 +8,6 @@#include "rxe_loc.h" /* info about object pools -* note that mr and mw share a single index space -* so that one can map an lkey to the correct type of object */ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES ] = { [RXE_TYPE_UC ] = {
@@ -50,15 +48,15 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = { }, [RXE_TYPE_MR] = { .name = "rxe-mr", - .size = sizeof(struct rxe_mem), - .cleanup = rxe_mem_cleanup, + .size = sizeof(struct rxe_mr), + .cleanup = rxe_mr_cleanup, .flags = RXE_POOL_INDEX, .max_index = RXE_MAX_MR_INDEX, .min_index = RXE_MIN_MR_INDEX, }, [RXE_TYPE_MW] = { .name = "rxe-mw", - .size = sizeof(struct rxe_mem), + .size = sizeof(struct rxe_mw), .flags = RXE_POOL_INDEX, .max_index = RXE_MAX_MW_INDEX, .min_index = RXE_MIN_MW_INDEX,
drivers/infiniband/sw/rxe/rxe_req.c +3 -3
@@ -465,7 +465,7 @@ static int fill_packet(struct rxe_qp *qp, struct rxe_send_wqe *wqe, }else { err =copy_data (qp->pd,0 , &wqe->dma,payload_addr (pkt), paylen, - from_mem_obj, + from_mr_obj, &crc);if (err)return err;
@@ -597,7 +597,7 @@ int rxe_requester(void *arg)if (wqe->mask &WR_REG_MASK ) {if (wqe->wr.opcode ==IB_WR_LOCAL_INV ) {struct rxe_dev *rxe =to_rdev (qp->ibqp.device); -struct rxe_mem *rmr; +struct rxe_mr *rmr; rmr =rxe_pool_get_index (&rxe->mr_pool, wqe->wr.ex.invalidate_rkey >>8 );
@@ -613,7 +613,7 @@ int rxe_requester(void *arg) wqe->state = wqe_state_done; wqe->status =IB_WC_SUCCESS ; }else if (wqe->wr.opcode ==IB_WR_REG_MR ) { -struct rxe_mem *rmr =to_rmr (wqe->wr.wr.reg.mr); +struct rxe_mr *rmr =to_rmr (wqe->wr.wr.reg.mr); rmr->state =RXE_MEM_STATE_VALID ; rmr->access = wqe->wr.wr.reg.access;
drivers/infiniband/sw/rxe/rxe_resp.c +15 -15
@@ -390,7 +390,7 @@ static enum resp_states check_length(struct rxe_qp *qp,static enum resp_states check_rkey (struct rxe_qp *qp,struct rxe_pkt_info *pkt) { -struct rxe_mem *mem =NULL ; +struct rxe_mr *mr =NULL ;u64 va;u32 rkey;u32 resid;
@@ -429,18 +429,18 @@ static enum resp_states check_rkey(struct rxe_qp *qp, resid = qp->resp.resid; pktlen =payload_size (pkt); - mem =lookup_mem (qp->pd, access, rkey, lookup_remote); -if (!mem) { + mr =lookup_mr (qp->pd, access, rkey, lookup_remote); +if (!mr) { state =RESPST_ERR_RKEY_VIOLATION ; goto err; } -if (unlikely (mem->state ==RXE_MEM_STATE_FREE )) { +if (unlikely (mr->state ==RXE_MEM_STATE_FREE )) { state =RESPST_ERR_RKEY_VIOLATION ; goto err; } -if (mem_check_range (mem, va, resid)) { +if (mr_check_range (mr, va, resid)) { state =RESPST_ERR_RKEY_VIOLATION ; goto err; }
@@ -468,12 +468,12 @@ static enum resp_states check_rkey(struct rxe_qp *qp,WARN_ON_ONCE (qp->resp.mr); - qp->resp.mr = mem; + qp->resp.mr = mr;return RESPST_EXECUTE ; err: -if (mem) -rxe_drop_ref (mem); +if (mr) +rxe_drop_ref (mr);return state; }
@@ -483,7 +483,7 @@ static enum resp_states send_data_in(struct rxe_qp *qp, void *data_addr,int err; err =copy_data (qp->pd,IB_ACCESS_LOCAL_WRITE , &qp->resp.wqe->dma, - data_addr, data_len, to_mem_obj,NULL ); + data_addr, data_len, to_mr_obj,NULL );if (unlikely (err))return (err == -ENOSPC ) ?RESPST_ERR_LENGTH :RESPST_ERR_MALFORMED_WQE ;
@@ -498,8 +498,8 @@ static enum resp_states write_data_in(struct rxe_qp *qp,int err;int data_len =payload_size (pkt); - err =rxe_mem_copy (qp->resp.mr, qp->resp.va,payload_addr (pkt), - data_len, to_mem_obj,NULL ); + err =rxe_mr_copy (qp->resp.mr, qp->resp.va,payload_addr (pkt), + data_len, to_mr_obj,NULL );if (err) { rc =RESPST_ERR_RKEY_VIOLATION ; goto out;
@@ -521,7 +521,7 @@ static enum resp_states process_atomic(struct rxe_qp *qp,u64 iova =atmeth_va (pkt);u64 *vaddr;enum resp_states ret; -struct rxe_mem *mr = qp->resp.mr; +struct rxe_mr *mr = qp->resp.mr;if (mr->state !=RXE_MEM_STATE_VALID ) { ret =RESPST_ERR_RKEY_VIOLATION ;
@@ -700,8 +700,8 @@ static enum resp_states read_reply(struct rxe_qp *qp,if (!skb)return RESPST_ERR_RNR ; - err =rxe_mem_copy (res->read.mr, res->read.va,payload_addr (&ack_pkt), - payload, from_mem_obj, &icrc); + err =rxe_mr_copy (res->read.mr, res->read.va,payload_addr (&ack_pkt), + payload, from_mr_obj, &icrc);if (err)pr_err ("Failed copying memory\n" );
@@ -883,7 +883,7 @@ static enum resp_states do_complete(struct rxe_qp *qp, }if (pkt->mask &RXE_IETH_MASK ) { -struct rxe_mem *rmr; +struct rxe_mr *rmr; wc->wc_flags |=IB_WC_WITH_INVALIDATE ; wc->ex.invalidate_rkey =ieth_rkey (pkt);
drivers/infiniband/sw/rxe/rxe_verbs.c +9 -9
@@ -867,7 +867,7 @@ static struct ib_mr *rxe_get_dma_mr(struct ib_pd *ibpd, int access) {struct rxe_dev *rxe =to_rdev (ibpd->device);struct rxe_pd *pd =to_rpd (ibpd); -struct rxe_mem *mr; +struct rxe_mr *mr; mr =rxe_alloc (&rxe->mr_pool);if (!mr)
@@ -875,7 +875,7 @@ static struct ib_mr *rxe_get_dma_mr(struct ib_pd *ibpd, int access)rxe_add_index (mr);rxe_add_ref (pd); -rxe_mem_init_dma (pd, access, mr); +rxe_mr_init_dma (pd, access, mr);return &mr->ibmr; }
@@ -889,7 +889,7 @@ static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd,int err;struct rxe_dev *rxe =to_rdev (ibpd->device);struct rxe_pd *pd =to_rpd (ibpd); -struct rxe_mem *mr; +struct rxe_mr *mr; mr =rxe_alloc (&rxe->mr_pool);if (!mr) {
@@ -901,7 +901,7 @@ static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd,rxe_add_ref (pd); - err =rxe_mem_init_user (pd, start, length, iova, + err =rxe_mr_init_user (pd, start, length, iova, access, udata, mr);if (err) goto err3;
@@ -918,7 +918,7 @@ static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd,static int rxe_dereg_mr (struct ib_mr *ibmr,struct ib_udata *udata) { -struct rxe_mem *mr =to_rmr (ibmr); +struct rxe_mr *mr =to_rmr (ibmr); mr->state =RXE_MEM_STATE_ZOMBIE ;rxe_drop_ref (mr->pd);
@@ -932,7 +932,7 @@ static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, {struct rxe_dev *rxe =to_rdev (ibpd->device);struct rxe_pd *pd =to_rpd (ibpd); -struct rxe_mem *mr; +struct rxe_mr *mr;int err;if (mr_type !=IB_MR_TYPE_MEM_REG )
@@ -948,7 +948,7 @@ static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,rxe_add_ref (pd); - err =rxe_mem_init_fast (pd, max_num_sg, mr); + err =rxe_mr_init_fast (pd, max_num_sg, mr);if (err) goto err2;
@@ -964,7 +964,7 @@ static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,static int rxe_set_page (struct ib_mr *ibmr,u64 addr) { -struct rxe_mem *mr =to_rmr (ibmr); +struct rxe_mr *mr =to_rmr (ibmr);struct rxe_map *map;struct rxe_phys_buf *buf;
@@ -984,7 +984,7 @@ static int rxe_set_page(struct ib_mr *ibmr, u64 addr)static int rxe_map_mr_sg (struct ib_mr *ibmr,struct scatterlist *sg,int sg_nents,unsigned int *sg_offset) { -struct rxe_mem *mr =to_rmr (ibmr); +struct rxe_mr *mr =to_rmr (ibmr);int n; mr->nbuf =0 ;
drivers/infiniband/sw/rxe/rxe_verbs.h +27 -24
106 lines not on this page · raw message
Signature · 1 line
2.25.1
PATCH v7 2/19 rdma_rxe: Enable MW objects
In reply to Bob PearsonChange parameters in rxe_param.h so that MAX_MW is the same as MAX_MR. Set device attribute in rxe.c so max_mw = MAX_MW.
--- drivers/infiniband/sw/rxe/rxe.c | 1 + drivers/infiniband/sw/rxe/rxe_param.h | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-)
drivers/infiniband/sw/rxe/rxe.c +1 -0
@@ -54,6 +54,7 @@ static void rxe_init_device_param(struct rxe_dev *rxe) rxe->attr.max_cq =RXE_MAX_CQ ; rxe->attr.max_cqe = (1 <<RXE_MAX_LOG_CQE ) -1 ; rxe->attr.max_mr =RXE_MAX_MR ; + rxe->attr.max_mw =RXE_MAX_MW ; rxe->attr.max_pd =RXE_MAX_PD ; rxe->attr.max_qp_rd_atom =RXE_MAX_QP_RD_ATOM ; rxe->attr.max_res_rd_atom =RXE_MAX_RES_RD_ATOM ;
drivers/infiniband/sw/rxe/rxe_param.h +6 -4
@@ -58,7 +58,8 @@ enum rxe_device_param {RXE_MAX_SGE_RD =32 ,RXE_MAX_CQ =16384 ,RXE_MAX_LOG_CQE =15 , -RXE_MAX_MR =256 *1024 , +RXE_MAX_MR =0x40000 , +RXE_MAX_MW =0x40000 ,RXE_MAX_PD =0x7ffc ,RXE_MAX_QP_RD_ATOM =128 , RXE_MAX_RES_RD_ATOM = 0x3f000,
@@ -87,9 +88,10 @@ enum rxe_device_param {RXE_MAX_SRQ_INDEX =0x00040000 ,RXE_MIN_MR_INDEX =0x00000001 , -RXE_MAX_MR_INDEX =0x00040000 , -RXE_MIN_MW_INDEX =0x00040001 , -RXE_MAX_MW_INDEX =0x00060000 , +RXE_MAX_MR_INDEX =RXE_MIN_MR_INDEX +RXE_MAX_MR -1 , +RXE_MIN_MW_INDEX =RXE_MIN_MR_INDEX +RXE_MAX_MR , +RXE_MAX_MW_INDEX =RXE_MIN_MW_INDEX +RXE_MAX_MW -1 , +RXE_MAX_PKT_PER_ACK =64 , RXE_MAX_UNACKED_PSNS = 128,
Signature · 1 line
2.25.1
PATCH v7 3/19 rdma_rxe: Let pools support both keys and indices
In reply to Bob PearsonAllow both indices and keys to exist for objects in pools. Previously you were limited to one or the other. This will support allowing the keys on MWs to change.
--- drivers/infiniband/sw/rxe/rxe_pool.c | 73 ++++++++++++++-------------- drivers/infiniband/sw/rxe/rxe_pool.h | 32 +++++++----- 2 files changed, 58 insertions(+), 47 deletions(-)
drivers/infiniband/sw/rxe/rxe_pool.c +37 -36
@@ -92,18 +92,18 @@ static int rxe_pool_init_index(struct rxe_pool *pool, u32 max, u32 min) goto out; } - pool->max_index = max; - pool->min_index = min; + pool->index.max_index = max; + pool->index.min_index = min; size =BITS_TO_LONGS (max - min +1 ) *sizeof (long ); - pool->table =kmalloc (size,GFP_KERNEL ); -if (!pool->table) { + pool->index.table =kmalloc (size,GFP_KERNEL ); +if (!pool->index.table) { err = -ENOMEM ; goto out; } - pool->table_size = size; -bitmap_zero (pool->table, max - min +1 ); + pool->index.table_size = size; +bitmap_zero (pool->index.table, max - min +1 ); out:return err;
@@ -125,7 +125,8 @@ int rxe_pool_init( pool->max_elem = max_elem; pool->elem_size =ALIGN (size,RXE_POOL_ALIGN ); pool->flags = rxe_type_info[type].flags; - pool->tree =RB_ROOT ; + pool->index.tree =RB_ROOT ; + pool->key.tree =RB_ROOT ; pool->cleanup = rxe_type_info[type].cleanup;atomic_set (&pool->num_elem,0 );
@@ -143,8 +144,8 @@ int rxe_pool_init( }if (rxe_type_info[type].flags &RXE_POOL_KEY ) { - pool->key_offset = rxe_type_info[type].key_offset; - pool->key_size = rxe_type_info[type].key_size; + pool->key.key_offset = rxe_type_info[type].key_offset; + pool->key.key_size = rxe_type_info[type].key_size; } pool->state =RXE_POOL_STATE_VALID ;
@@ -158,7 +159,7 @@ static void rxe_pool_release(struct kref *kref) struct rxe_pool *pool = container_of(kref, struct rxe_pool, ref_cnt); pool->state =RXE_POOL_STATE_INVALID ; -kfree (pool->table); +kfree (pool->index.table); }static void rxe_pool_put (struct rxe_pool *pool)
@@ -183,27 +184,27 @@ void rxe_pool_cleanup(struct rxe_pool *pool)static u32 alloc_index (struct rxe_pool *pool) {u32 index; -u32 range = pool->max_index - pool->min_index +1 ; +u32 range = pool->index.max_index - pool->index.min_index +1 ; - index =find_next_zero_bit (pool->table, range, pool->last); + index =find_next_zero_bit (pool->index.table, range, pool->index.last);if (index >= range) - index =find_first_zero_bit (pool->table, range); + index =find_first_zero_bit (pool->index.table, range);WARN_ON_ONCE (index >= range); -set_bit (index, pool->table); - pool->last = index; -return index + pool->min_index; +set_bit (index, pool->index.table); + pool->index.last = index; +return index + pool->index.min_index; }static void insert_index (struct rxe_pool *pool,struct rxe_pool_entry *new) { -struct rb_node **link = &pool->tree.rb_node; +struct rb_node **link = &pool->index.tree.rb_node;struct rb_node *parent =NULL ;struct rxe_pool_entry *elem;while (*link) { parent = *link; - elem = rb_entry(parent, struct rxe_pool_entry, node); + elem = rb_entry(parent, struct rxe_pool_entry, index_node);if (elem->index == new->index) {pr_warn ("element already exists!\n" );
@@ -216,25 +217,25 @@ static void insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new) link = &(*link)->rb_right; } -rb_link_node (&new->node, parent, link); -rb_insert_color (&new->node, &pool->tree); +rb_link_node (&new->index_node, parent, link); +rb_insert_color (&new->index_node, &pool->index.tree); out:return ; }static void insert_key (struct rxe_pool *pool,struct rxe_pool_entry *new) { -struct rb_node **link = &pool->tree.rb_node; +struct rb_node **link = &pool->key.tree.rb_node;struct rb_node *parent =NULL ;struct rxe_pool_entry *elem;int cmp;while (*link) { parent = *link; - elem = rb_entry(parent, struct rxe_pool_entry, node); + elem = rb_entry(parent, struct rxe_pool_entry, key_node); - cmp =memcmp ((u8 *)elem + pool->key_offset, - (u8 *)new + pool->key_offset, pool->key_size); + cmp =memcmp ((u8 *)elem + pool->key.key_offset, + (u8 *)new + pool->key.key_offset, pool->key.key_size);if (cmp ==0 ) {pr_warn ("key already exists!\n" );
@@ -247,8 +248,8 @@ static void insert_key(struct rxe_pool *pool, struct rxe_pool_entry *new) link = &(*link)->rb_right; } -rb_link_node (&new->node, parent, link); -rb_insert_color (&new->node, &pool->tree); +rb_link_node (&new->key_node, parent, link); +rb_insert_color (&new->key_node, &pool->key.tree); out:return ; }
@@ -260,7 +261,7 @@ void rxe_add_key(void *arg, void *key)unsigned long flags;write_lock_irqsave (&pool->pool_lock, flags); -memcpy ((u8 *)elem + pool->key_offset, key, pool->key_size); +memcpy ((u8 *)elem + pool->key.key_offset, key, pool->key.key_size);insert_key (pool, elem);write_unlock_irqrestore (&pool->pool_lock, flags); }
@@ -272,7 +273,7 @@ void rxe_drop_key(void *arg)unsigned long flags;write_lock_irqsave (&pool->pool_lock, flags); -rb_erase (&elem->node, &pool->tree); +rb_erase (&elem->key_node, &pool->key.tree);write_unlock_irqrestore (&pool->pool_lock, flags); }
@@ -295,8 +296,8 @@ void rxe_drop_index(void *arg)unsigned long flags;write_lock_irqsave (&pool->pool_lock, flags); -clear_bit (elem->index - pool->min_index, pool->table); -rb_erase (&elem->node, &pool->tree); +clear_bit (elem->index - pool->index.min_index, pool->index.table); +rb_erase (&elem->index_node, &pool->index.tree);write_unlock_irqrestore (&pool->pool_lock, flags); }
@@ -400,10 +401,10 @@ void *rxe_pool_get_index(struct rxe_pool *pool, u32 index)if (pool->state !=RXE_POOL_STATE_VALID ) goto out; - node = pool->tree.rb_node; + node = pool->index.tree.rb_node;while (node) { - elem = rb_entry(node, struct rxe_pool_entry, node); + elem = rb_entry(node, struct rxe_pool_entry, index_node);if (elem->index > index) node = node->rb_left;
@@ -432,13 +433,13 @@ void *rxe_pool_get_key(struct rxe_pool *pool, void *key)if (pool->state !=RXE_POOL_STATE_VALID ) goto out; - node = pool->tree.rb_node; + node = pool->key.tree.rb_node;while (node) { - elem = rb_entry(node, struct rxe_pool_entry, node); + elem = rb_entry(node, struct rxe_pool_entry, key_node); - cmp =memcmp ((u8 *)elem + pool->key_offset, - key, pool->key_size); + cmp =memcmp ((u8 *)elem + pool->key.key_offset, + key, pool->key.key_size);if (cmp >0 ) node = node->rb_left;
drivers/infiniband/sw/rxe/rxe_pool.h +21 -11
@@ -56,8 +56,11 @@ struct rxe_pool_entry {struct kref ref_cnt;struct list_head list; -/* only used if indexed or keyed */ -struct rb_node node; +/* only used if keyed */ +struct rb_node key_node; + +/* only used if indexed */ +struct rb_node index_node;u32 index; };
@@ -74,15 +77,22 @@ struct rxe_pool {unsigned int max_elem;atomic_t num_elem; -/* only used if indexed or keyed */ -struct rb_root tree; -unsigned long *table; -size_t table_size; -u32 max_index; -u32 min_index; -u32 last; -size_t key_offset; -size_t key_size; +/* only used if indexed */ +struct { +struct rb_root tree; +unsigned long *table; +size_t table_size; +u32 last; +u32 max_index; +u32 min_index; + } index; + +/* only used if keyed */ +struct { +struct rb_root tree; +size_t key_offset; +size_t key_size; + } key; };/* initialize a pool of objects with given limit on
Signature · 1 line
2.25.1
PATCH v7 4/19 rdma_rxe: make pool return values position independent
In reply to Bob Pearsonrxe pool manages objects which are based on a base type of rxe_pool_entry. Once apon a time these were always the first entry in the rxe object structs. Now the pool entries (usually named elem or pelem) occur in more than one position. In order to make the return values from lookup and allocate routines work independently of the position of the pelem the offset of pelem is added to the rxe_type_info struct and used to compute the return value from these routines.
--- drivers/infiniband/sw/rxe/rxe_mcast.c | 4 +-- drivers/infiniband/sw/rxe/rxe_mr.c | 6 ++-- drivers/infiniband/sw/rxe/rxe_pool.c | 40 +++++++++++++++++++++------ drivers/infiniband/sw/rxe/rxe_pool.h | 5 ++-- drivers/infiniband/sw/rxe/rxe_recv.c | 6 ++-- drivers/infiniband/sw/rxe/rxe_req.c | 2 +- drivers/infiniband/sw/rxe/rxe_resp.c | 2 +- 7 files changed, 44 insertions(+), 21 deletions(-)
drivers/infiniband/sw/rxe/rxe_mcast.c +2 -2
@@ -18,7 +18,7 @@ int rxe_mcast_get_grp(struct rxe_dev *rxe, union ib_gid *mgid, goto err1; } - grp =rxe_pool_get_key (&rxe->mc_grp_pool, mgid); + grp =rxe_get_key (&rxe->mc_grp_pool, mgid);if (grp) goto done;
@@ -98,7 +98,7 @@ int rxe_mcast_drop_grp_elem(struct rxe_dev *rxe, struct rxe_qp *qp,struct rxe_mc_grp *grp;struct rxe_mc_elem *elem, *tmp; - grp =rxe_pool_get_key (&rxe->mc_grp_pool, mgid); + grp =rxe_get_key (&rxe->mc_grp_pool, mgid);if (!grp) goto err1;
drivers/infiniband/sw/rxe/rxe_mr.c +3 -3
@@ -10,7 +10,7 @@/* * lfsr (linear feedback shift register) with period 255 */ -static u8 rxe_get_key (void ) +static u8 get_key (void ) {static u32 key =1 ;
@@ -49,7 +49,7 @@ int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)static void rxe_mr_init (int access,struct rxe_mr *mr) { -u32 lkey = mr->pelem.index <<8 |rxe_get_key (); +u32 lkey = mr->pelem.index <<8 |get_key ();u32 rkey = (access &IB_ACCESS_REMOTE ) ? lkey :0 ;if (mr->pelem.pool->type ==RXE_TYPE_MR ) {
@@ -544,7 +544,7 @@ struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,struct rxe_dev *rxe =to_rdev (pd->ibpd.device);int index = key >>8 ; - mr =rxe_pool_get_index (&rxe->mr_pool, index); + mr =rxe_get_index (&rxe->mr_pool, index);if (!mr)return NULL ;
drivers/infiniband/sw/rxe/rxe_pool.c +31 -9
@@ -13,21 +13,25 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = { [RXE_TYPE_UC] = { .name = "rxe-uc", .size = sizeof(struct rxe_ucontext), + .elem_offset = offsetof(struct rxe_ucontext, pelem), .flags = RXE_POOL_NO_ALLOC, }, [RXE_TYPE_PD] = { .name = "rxe-pd", .size = sizeof(struct rxe_pd), + .elem_offset = offsetof(struct rxe_pd, pelem), .flags = RXE_POOL_NO_ALLOC, }, [RXE_TYPE_AH] = { .name = "rxe-ah", .size = sizeof(struct rxe_ah), + .elem_offset = offsetof(struct rxe_ah, pelem), .flags = RXE_POOL_ATOMIC | RXE_POOL_NO_ALLOC, }, [RXE_TYPE_SRQ] = { .name = "rxe-srq", .size = sizeof(struct rxe_srq), + .elem_offset = offsetof(struct rxe_srq, pelem), .flags = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC, .min_index = RXE_MIN_SRQ_INDEX, .max_index = RXE_MAX_SRQ_INDEX,
@@ -35,6 +39,7 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = { [RXE_TYPE_QP] = { .name = "rxe-qp", .size = sizeof(struct rxe_qp), + .elem_offset = offsetof(struct rxe_qp, pelem), .cleanup = rxe_qp_cleanup, .flags = RXE_POOL_INDEX, .min_index = RXE_MIN_QP_INDEX,
@@ -43,12 +48,14 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = { [RXE_TYPE_CQ] = { .name = "rxe-cq", .size = sizeof(struct rxe_cq), + .elem_offset = offsetof(struct rxe_cq, pelem), .flags = RXE_POOL_NO_ALLOC, .cleanup = rxe_cq_cleanup, }, [RXE_TYPE_MR] = { .name = "rxe-mr", .size = sizeof(struct rxe_mr), + .elem_offset = offsetof(struct rxe_mr, pelem), .cleanup = rxe_mr_cleanup, .flags = RXE_POOL_INDEX, .max_index = RXE_MAX_MR_INDEX,
@@ -57,6 +64,7 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = { [RXE_TYPE_MW] = { .name = "rxe-mw", .size = sizeof(struct rxe_mw), + .elem_offset = offsetof(struct rxe_mw, pelem), .flags = RXE_POOL_INDEX, .max_index = RXE_MAX_MW_INDEX, .min_index = RXE_MIN_MW_INDEX,
@@ -64,6 +72,7 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = { [RXE_TYPE_MC_GRP] = { .name = "rxe-mc_grp", .size = sizeof(struct rxe_mc_grp), + .elem_offset = offsetof(struct rxe_mc_grp, pelem), .cleanup = rxe_mc_cleanup, .flags = RXE_POOL_KEY, .key_offset = offsetof(struct rxe_mc_grp, mgid),
@@ -72,6 +81,7 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = { [RXE_TYPE_MC_ELEM ] = { .name ="rxe-mc_elem" , .size =sizeof (struct rxe_mc_elem ), + .elem_offset = offsetof(struct rxe_mc_elem , pelem), .flags =RXE_POOL_ATOMIC , }, };
@@ -304,6 +314,8 @@ void rxe_drop_index(void *arg)void *rxe_alloc (struct rxe_pool *pool) {struct rxe_pool_entry *elem; +struct rxe_type_info *info = &rxe_type_info[pool->type]; +u8 *obj =NULL ;unsigned long flags;might_sleep_if (!(pool->flags &RXE_POOL_ATOMIC ));
@@ -330,8 +342,9 @@ void *rxe_alloc(struct rxe_pool *pool) elem->pool = pool;kref_init (&elem->ref_cnt); + obj = (u8 *)elem - info->elem_offset; -return elem; +return obj; out_cnt:atomic_dec (&pool->num_elem);
@@ -390,10 +403,12 @@ void rxe_elem_release(struct kref *kref)rxe_pool_put (pool); } -void *rxe_pool_get_index (struct rxe_pool *pool,u32 index) +void *rxe_get_index (struct rxe_pool *pool,u32 index) {struct rb_node *node =NULL ;struct rxe_pool_entry *elem =NULL ; +struct rxe_type_info *info = &rxe_type_info[pool->type]; +u8 *obj =NULL ;unsigned long flags;read_lock_irqsave (&pool->pool_lock, flags);
@@ -410,21 +425,25 @@ void *rxe_pool_get_index(struct rxe_pool *pool, u32 index) node = node->rb_left; else if (elem->index < index) node = node->rb_right; - else { -kref_get (&elem->ref_cnt); + else break; - } } +if (node) { +kref_get (&elem->ref_cnt); + obj = (u8 *)elem - info->elem_offset; + } out:read_unlock_irqrestore (&pool->pool_lock, flags); -return node ? elem :NULL ; +return obj; } -void *rxe_pool_get_key (struct rxe_pool *pool,void *key) +void *rxe_get_key (struct rxe_pool *pool,void *key) {struct rb_node *node =NULL ;struct rxe_pool_entry *elem =NULL ; +struct rxe_type_info *info = &rxe_type_info[pool->type]; +u8 *obj =NULL ;int cmp;unsigned long flags;
@@ -449,10 +468,13 @@ void *rxe_pool_get_key(struct rxe_pool *pool, void *key)break ; } -if (node) +if (node) {kref_get (&elem->ref_cnt); + obj = (u8 *)elem - info->elem_offset; + } + out:read_unlock_irqrestore (&pool->pool_lock, flags); -return node ? elem :NULL ; +return obj; }
drivers/infiniband/sw/rxe/rxe_pool.h +3 -2
@@ -36,6 +36,7 @@ struct rxe_pool_entry;struct rxe_type_info {const char *name;size_t size; +size_t elem_offset;void (*cleanup)(struct rxe_pool_entry *obj);enum rxe_pool_flags flags;u32 max_index;
@@ -128,10 +129,10 @@ void rxe_add_key(void *elem, void *key);void rxe_drop_key (void *elem);/* lookup an indexed object from index. takes a reference on object */ -void *rxe_pool_get_index (struct rxe_pool *pool,u32 index); +void *rxe_get_index (struct rxe_pool *pool,u32 index);/* lookup keyed object from key. takes a reference on the object */ -void *rxe_pool_get_key (struct rxe_pool *pool,void *key); +void *rxe_get_key (struct rxe_pool *pool,void *key);/* cleanup an object when all references are dropped */ void rxe_elem_release (struct kref *kref);
drivers/infiniband/sw/rxe/rxe_recv.c +3 -3
@@ -185,7 +185,7 @@ static int hdr_check(struct rxe_pkt_info *pkt)if (qpn !=IB_MULTICAST_QPN ) { index = (qpn ==1 ) ? port->qp_gsi_index : qpn; - qp =rxe_pool_get_index (&rxe->qp_pool, index); + qp =rxe_get_index (&rxe->qp_pool, index);if (unlikely (!qp)) {pr_warn_ratelimited ("no qp matches qpn 0x%x\n" , qpn); goto err1;
@@ -242,7 +242,7 @@ static void rxe_rcv_mcast_pkt(struct rxe_dev *rxe, struct sk_buff *skb)memcpy (&dgid, &ipv6_hdr (skb)->daddr,sizeof (dgid));/* lookup mcast group corresponding to mgid, takes a ref */ - mcg =rxe_pool_get_key (&rxe->mc_grp_pool, &dgid); + mcg =rxe_get_key (&rxe->mc_grp_pool, &dgid);if (!mcg) goto err1;/* mcast group not registered */
@@ -274,7 +274,7 @@ static void rxe_rcv_mcast_pkt(struct rxe_dev *rxe, struct sk_buff *skb)spin_unlock_bh (&mcg->mcg_lock); -rxe_drop_ref (mcg);/* drop ref from rxe_pool_get_key. */ +rxe_drop_ref (mcg);/* drop ref from rxe_get_key. */ err1:kfree_skb (skb);
drivers/infiniband/sw/rxe/rxe_req.c +1 -1
@@ -599,7 +599,7 @@ int rxe_requester(void *arg)struct rxe_dev *rxe =to_rdev (qp->ibqp.device);struct rxe_mr *rmr; - rmr =rxe_pool_get_index (&rxe->mr_pool, + rmr =rxe_get_index (&rxe->mr_pool, wqe->wr.ex.invalidate_rkey >>8 );if (!rmr) { pr_err("No mr for key %#x\n",
drivers/infiniband/sw/rxe/rxe_resp.c +1 -1
@@ -888,7 +888,7 @@ static enum resp_states do_complete(struct rxe_qp *qp, wc->wc_flags |=IB_WC_WITH_INVALIDATE ; wc->ex.invalidate_rkey =ieth_rkey (pkt); - rmr =rxe_pool_get_index (&rxe->mr_pool, + rmr =rxe_get_index (&rxe->mr_pool, wc->ex.invalidate_rkey >>8 );if (unlikely (!rmr)) { pr_err("Bad rkey %#x invalidation\n",
Signature · 1 line
2.25.1
PATCH v7 5/19 rdma_rxe: remove void * parameters in pool APIs
In reply to Bob PearsonMore work to make rxe pools type safe.
Replaced routines that took void * as parameters and and replaced them with rxe_pool_entry *elem. Then added macros to convert objects to pool entries. This makes these routines type safe against moving the pool entries around in the objects.
--- drivers/infiniband/sw/rxe/rxe_pool.c | 14 +++++-------- drivers/infiniband/sw/rxe/rxe_pool.h | 29 ++++++++++++++------------- drivers/infiniband/sw/rxe/rxe_verbs.c | 10 ++++----- 3 files changed, 25 insertions(+), 28 deletions(-)
drivers/infiniband/sw/rxe/rxe_pool.c +5 -9
@@ -264,9 +264,8 @@ static void insert_key(struct rxe_pool *pool, struct rxe_pool_entry *new)return ; } -void rxe_add_key (void *arg,void *key) +void __rxe_add_key (struct rxe_pool_entry *elem,void *key) { -struct rxe_pool_entry *elem = arg;struct rxe_pool *pool = elem->pool;unsigned long flags;
@@ -276,9 +275,8 @@ void rxe_add_key(void *arg, void *key)write_unlock_irqrestore (&pool->pool_lock, flags); } -void rxe_drop_key (void *arg) +void __rxe_drop_key (struct rxe_pool_entry *elem) { -struct rxe_pool_entry *elem = arg;struct rxe_pool *pool = elem->pool;unsigned long flags;
@@ -287,9 +285,8 @@ void rxe_drop_key(void *arg)write_unlock_irqrestore (&pool->pool_lock, flags); } -void rxe_add_index (void *arg) +void __rxe_add_index (struct rxe_pool_entry *elem) { -struct rxe_pool_entry *elem = arg;struct rxe_pool *pool = elem->pool;unsigned long flags;
@@ -299,9 +296,8 @@ void rxe_add_index(void *arg)write_unlock_irqrestore (&pool->pool_lock, flags); } -void rxe_drop_index (void *arg) +void __rxe_drop_index (struct rxe_pool_entry *elem) { -struct rxe_pool_entry *elem = arg;struct rxe_pool *pool = elem->pool;unsigned long flags;
@@ -354,7 +350,7 @@ void *rxe_alloc(struct rxe_pool *pool)return NULL ; } -int rxe_add_to_pool (struct rxe_pool *pool,struct rxe_pool_entry *elem) +int __rxe_add_to_pool (struct rxe_pool *pool,struct rxe_pool_entry *elem) {unsigned long flags;
drivers/infiniband/sw/rxe/rxe_pool.h +15 -14
@@ -109,24 +109,25 @@ void rxe_pool_cleanup(struct rxe_pool *pool);/* allocate an object from pool */ void *rxe_alloc (struct rxe_pool *pool); -/* connect already allocated object to pool */ -int rxe_add_to_pool (struct rxe_pool *pool,struct rxe_pool_entry *elem); +int __rxe_add_to_pool (struct rxe_pool *pool,struct rxe_pool_entry *elem); -/* assign an index to an indexed object and insert object into -* pool's rb tree -*/ -void rxe_add_index (void *elem); +#define rxe_add_to_pool (pool, obj) __rxe_add_to_pool(pool, &(obj)->pelem) -/* drop an index and remove object from rb tree */ -void rxe_drop_index (void *elem); +void __rxe_add_index (struct rxe_pool_entry *elem); -/* assign a key to a keyed object and insert object into -* pool's rb tree -*/ -void rxe_add_key (void *elem,void *key); +#define rxe_add_index (obj) __rxe_add_index(&(obj)->pelem) + +void __rxe_drop_index (struct rxe_pool_entry *elem); + +#define rxe_drop_index (obj) __rxe_drop_index(&(obj)->pelem) + +void __rxe_add_key (struct rxe_pool_entry *elem,void *key); + +#define rxe_add_key (obj, key) __rxe_add_key(&(obj)->pelem, key) + +void __rxe_drop_key (struct rxe_pool_entry *elem); -/* remove elem from rb tree */ -void rxe_drop_key (void *elem); +#define rxe_drop_key (obj) __rxe_drop_key(&(obj)->pelem)/* lookup an indexed object from index. takes a reference on object */ void *rxe_get_index (struct rxe_pool *pool,u32 index);
drivers/infiniband/sw/rxe/rxe_verbs.c +5 -5
@@ -111,7 +111,7 @@ static int rxe_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)struct rxe_dev *rxe =to_rdev (uctx->device);struct rxe_ucontext *uc =to_ruc (uctx); -return rxe_add_to_pool (&rxe->uc_pool, &uc->pelem); +return rxe_add_to_pool (&rxe->uc_pool, uc); }static void rxe_dealloc_ucontext (struct ib_ucontext *ibuc)
@@ -145,7 +145,7 @@ static int rxe_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)struct rxe_dev *rxe =to_rdev (ibpd->device);struct rxe_pd *pd =to_rpd (ibpd); -return rxe_add_to_pool (&rxe->pd_pool, &pd->pelem); +return rxe_add_to_pool (&rxe->pd_pool, pd); }static int rxe_dealloc_pd (struct ib_pd *ibpd,struct ib_udata *udata)
@@ -169,7 +169,7 @@ static int rxe_create_ah(struct ib_ah *ibah,if (err)return err; - err =rxe_add_to_pool (&rxe->ah_pool, &ah->pelem); + err =rxe_add_to_pool (&rxe->ah_pool, ah);if (err)return err;
@@ -275,7 +275,7 @@ static int rxe_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init,if (err) goto err1; - err =rxe_add_to_pool (&rxe->srq_pool, &srq->pelem); + err =rxe_add_to_pool (&rxe->srq_pool, srq);if (err) goto err1;
@@ -776,7 +776,7 @@ static int rxe_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,if (err)return err; -return rxe_add_to_pool (&rxe->cq_pool, &cq->pelem); +return rxe_add_to_pool (&rxe->cq_pool, cq); }static int rxe_destroy_cq (struct ib_cq *ibcq,struct ib_udata *udata)
Signature · 1 line
2.25.1
PATCH v7 6/19 rdma_rxe: add alloc_mw and dealloc_mw verbs
In reply to Bob Pearsonv7:
- Adjust for mw being allocated in core.
- Made MW NO_ALLOC type in pool
- Rearranged rxe_mw struct to make ibmw first
v6:
- Add a new file focused on memory windows, rxe_mw.c.
- Add alloc_mw and dealloc_mw verbs and added them to the list of supported user space verbs.
- Added indices AND keys for MW and MR.
- Add indices to user api response for MW and MR
- Replaced lookup by index with lookup by key in rxe_req.c and rxe_resp.c
--- drivers/infiniband/sw/rxe/Makefile | 1 + drivers/infiniband/sw/rxe/rxe_loc.h | 7 +++ drivers/infiniband/sw/rxe/rxe_mr.c | 80 +++++++++++-------------- drivers/infiniband/sw/rxe/rxe_mw.c | 85 +++++++++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_pool.c | 50 ++++++++++------ drivers/infiniband/sw/rxe/rxe_pool.h | 2 +- drivers/infiniband/sw/rxe/rxe_req.c | 4 +- drivers/infiniband/sw/rxe/rxe_resp.c | 4 +- drivers/infiniband/sw/rxe/rxe_verbs.c | 53 +++++++++++------ drivers/infiniband/sw/rxe/rxe_verbs.h | 10 +++- include/uapi/rdma/rdma_user_rxe.h | 10 ++++ 11 files changed, 219 insertions(+), 87 deletions(-)
drivers/infiniband/sw/rxe/Makefile +1 -0
@@ -15,6 +15,7 @@ rdma_rxe-y := \ rxe_qp.o \ rxe_cq.o \ rxe_mr.o \ + rxe_mw.o \ rxe_opcode.o \ rxe_mmap.o \ rxe_icrc.o \
drivers/infiniband/sw/rxe/rxe_loc.h +7 -0
@@ -109,6 +109,13 @@ void rxe_mr_cleanup(struct rxe_pool_entry *arg);int advance_dma_data (struct rxe_dma_info *dma,unsigned int length); +/* rxe_mw.c */ +int rxe_alloc_mw (struct ib_mw *ibmw,struct ib_udata *udata); + +int rxe_dealloc_mw (struct ib_mw *ibmw); + +void rxe_mw_cleanup (struct rxe_pool_entry *arg); +/* rxe_net.c */ void rxe_loopback (struct sk_buff *skb);int rxe_send (struct rxe_pkt_info *pkt,struct sk_buff *skb);
drivers/infiniband/sw/rxe/rxe_mr.c +35 -45
115 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_mw.c new file +85 -0
85 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_pool.c +32 -18
139 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_pool.h +1 -1
@@ -121,7 +121,7 @@ void __rxe_drop_index(struct rxe_pool_entry *elem);#define rxe_drop_index (obj) __rxe_drop_index(&(obj)->pelem) -void __rxe_add_key (struct rxe_pool_entry *elem,void *key); +int __rxe_add_key (struct rxe_pool_entry *elem,void *key);#define rxe_add_key (obj, key) __rxe_add_key(&(obj)->pelem, key)
drivers/infiniband/sw/rxe/rxe_req.c +2 -2
@@ -599,8 +599,8 @@ int rxe_requester(void *arg)struct rxe_dev *rxe =to_rdev (qp->ibqp.device);struct rxe_mr *rmr; - rmr =rxe_get_index (&rxe->mr_pool, - wqe->wr.ex.invalidate_rkey >>8 ); + rmr =rxe_get_key (&rxe->mr_pool, + &wqe->wr.ex.invalidate_rkey);if (!rmr) {pr_err ("No mr for key %#x\n" , wqe->wr.ex.invalidate_rkey);
drivers/infiniband/sw/rxe/rxe_resp.c +2 -2
@@ -888,8 +888,8 @@ static enum resp_states do_complete(struct rxe_qp *qp, wc->wc_flags |=IB_WC_WITH_INVALIDATE ; wc->ex.invalidate_rkey =ieth_rkey (pkt); - rmr =rxe_get_index (&rxe->mr_pool, - wc->ex.invalidate_rkey >>8 ); + rmr =rxe_get_key (&rxe->mr_pool, + &wc->ex.invalidate_rkey);if (unlikely (!rmr)) {pr_err ("Bad rkey %#x invalidation\n" , wc->ex.invalidate_rkey);
drivers/infiniband/sw/rxe/rxe_verbs.c +35 -18
123 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_verbs.h +9 -1
@@ -319,9 +319,12 @@ struct rxe_mr {struct rxe_map **map; }; +/* use high order bit to separate MW and MR rkeys */ +#define IS_MW (1 << 31) +struct rxe_mw { -struct rxe_pool_entry pelem;struct ib_mw ibmw; +struct rxe_pool_entry pelem;struct rxe_qp *qp;/* type 2B only */ struct rxe_mr *mr;spinlock_t lock;
@@ -441,6 +444,11 @@ static inline struct rxe_mr *to_rmr(struct ib_mr *mr) return mr ? container_of(mr, struct rxe_mr, ibmr) : NULL; } +static inline struct rxe_mw *to_rmw (struct ib_mw *mw) +{ + return mw ? container_of(mw, struct rxe_mw, ibmw) : NULL; +} +int rxe_register_device (struct rxe_dev *rxe,const char *ibdev_name);void rxe_mc_cleanup (struct rxe_pool_entry *arg);
include/uapi/rdma/rdma_user_rxe.h +10 -0
@@ -175,4 +175,14 @@ struct rxe_modify_srq_cmd { __aligned_u64 mmap_info_addr; }; +struct rxe_reg_mr_resp { +__u32 index; +__u32 reserved; +}; + +struct rxe_alloc_mw_resp { +__u32 index; +__u32 reserved; +}; +#endif /* RDMA_USER_RXE_H */
Signature · 1 line
2.25.1
PATCH v7 7/19 rdma_rxe: Add bind_mw and invalidate_mw verbs
In reply to Bob Pearsonv7:
- adjust for new pool api changes
v6:
- Add code to implement ibv_bind_mw (for type 1 MWs) and post send queue bind_mw (for type 2 MWs).
- Add code to implement local (post send) and remote (send with invalidate) invalidate operations.
- Add rules checking for MW operations from IBA.
--- drivers/infiniband/sw/rxe/rxe_comp.c | 1 + drivers/infiniband/sw/rxe/rxe_loc.h | 6 + drivers/infiniband/sw/rxe/rxe_mr.c | 13 +- drivers/infiniband/sw/rxe/rxe_mw.c | 289 ++++++++++++++++++++++++- drivers/infiniband/sw/rxe/rxe_opcode.c | 11 +- drivers/infiniband/sw/rxe/rxe_opcode.h | 1 - drivers/infiniband/sw/rxe/rxe_req.c | 134 +++++++++--- drivers/infiniband/sw/rxe/rxe_resp.c | 81 +++++-- drivers/infiniband/sw/rxe/rxe_verbs.c | 2 +- drivers/infiniband/sw/rxe/rxe_verbs.h | 7 + include/uapi/rdma/rdma_user_rxe.h | 34 ++- 11 files changed, 512 insertions(+), 67 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +1 -0
@@ -103,6 +103,7 @@ static enum ib_wc_opcode wr_to_wc_opcode(enum ib_wr_opcode opcode)case IB_WR_RDMA_READ_WITH_INV :return IB_WC_RDMA_READ ;case IB_WR_LOCAL_INV :return IB_WC_LOCAL_INV ;case IB_WR_REG_MR :return IB_WC_REG_MR ; +case IB_WR_BIND_MW :return IB_WC_BIND_MW ;default :return 0xff ;
drivers/infiniband/sw/rxe/rxe_loc.h +6 -0
@@ -109,6 +109,8 @@ void rxe_mr_cleanup(struct rxe_pool_entry *arg);int advance_dma_data (struct rxe_dma_info *dma,unsigned int length); +int rxe_invalidate_mr (struct rxe_qp *qp,struct rxe_mr *mr); +/* rxe_mw.c */ int rxe_alloc_mw (struct ib_mw *ibmw,struct ib_udata *udata);
@@ -116,6 +118,10 @@ int rxe_dealloc_mw(struct ib_mw *ibmw);void rxe_mw_cleanup (struct rxe_pool_entry *arg); +int rxe_bind_mw (struct rxe_qp *qp,struct rxe_send_wqe *wqe); + +int rxe_invalidate_mw (struct rxe_qp *qp,struct rxe_mw *mw); +/* rxe_net.c */ void rxe_loopback (struct sk_buff *skb);int rxe_send (struct rxe_pkt_info *pkt,struct sk_buff *skb);
drivers/infiniband/sw/rxe/rxe_mr.c +12 -1
@@ -532,12 +532,23 @@ struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,return mr; } +int rxe_invalidate_mr (struct rxe_qp *qp,struct rxe_mr *mr) +{ +/* TODO there are API rules being ignored here +* cleanup later. Current project is not trying +* to fix MR +*/ + mr->state =RXE_MEM_STATE_FREE ; +return 0 ; +} +void rxe_mr_cleanup (struct rxe_pool_entry *elem) {struct rxe_mr *mr =container_of (elem,typeof (*mr), pelem);int i; -ib_umem_release (mr->umem); +if (mr->umem) +ib_umem_release (mr->umem);if (mr->map) {for (i =0 ; i < mr->num_map; i++)
drivers/infiniband/sw/rxe/rxe_mw.c +282 -7
321 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_opcode.c +9 -2
@@ -87,13 +87,20 @@ struct rxe_wr_opcode_info rxe_wr_opcode_info[] = { [IB_WR_LOCAL_INV ] = { .name ="IB_WR_LOCAL_INV" , .mask = { - [IB_QPT_RC ] =WR_REG_MASK , + [IB_QPT_RC ] =WR_LOCAL_MASK , }, }, [IB_WR_REG_MR ] = { .name ="IB_WR_REG_MR" , .mask = { - [IB_QPT_RC ] =WR_REG_MASK , + [IB_QPT_RC ] =WR_LOCAL_MASK , + }, + }, + [IB_WR_BIND_MW ] = { + .name ="IB_WR_BIND_MW" , + .mask = { + [IB_QPT_RC ] =WR_LOCAL_MASK , + [IB_QPT_UC ] =WR_LOCAL_MASK , }, }, };
drivers/infiniband/sw/rxe/rxe_opcode.h +0 -1
@@ -20,7 +20,6 @@ enum rxe_wr_mask {WR_READ_MASK =BIT (3 ),WR_WRITE_MASK =BIT (4 ),WR_LOCAL_MASK =BIT (5 ), -WR_REG_MASK =BIT (6 ),WR_READ_OR_WRITE_MASK =WR_READ_MASK |WR_WRITE_MASK , WR_READ_WRITE_OR_SEND_MASK = WR_READ_OR_WRITE_MASK | WR_SEND_MASK,
drivers/infiniband/sw/rxe/rxe_req.c +98 -36
207 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_resp.c +63 -18
128 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_verbs.c +1 -1
@@ -577,7 +577,7 @@ static int init_send_wqe(struct rxe_qp *qp, const struct ib_send_wr *ibwr, p += sge->length; } - }else if (mask &WR_REG_MASK ) { + }else if (mask &WR_LOCAL_MASK ) { wqe->mask = mask; wqe->state = wqe_state_posted;return 0 ;
drivers/infiniband/sw/rxe/rxe_verbs.h +7 -0
@@ -316,9 +316,16 @@ struct rxe_mr {u32 max_buf;u32 num_map; +atomic_t num_mw; +struct rxe_map **map; }; +enum rxe_send_flags { +/* flag indicaes bind call came through verbs API */ +RXE_BIND_MW = (1 <<0 ), +}; +/* use high order bit to separate MW and MR rkeys */ #define IS_MW (1 << 31)
include/uapi/rdma/rdma_user_rxe.h +33 -1
@@ -93,7 +93,39 @@ struct rxe_send_wr {__u32 remote_qkey;__u16 pkey_index; } ud; -/* reg is only used by the kernel and is not part of the uapi */ +struct { +__aligned_u64 addr; +__aligned_u64 length; +union { +__u32 mr_index; +__aligned_u64 reserved1; + }; +union { +__u32 mw_index; +__aligned_u64 reserved2; + }; +__u32 rkey; +__u32 access; +__u32 flags; + } umw; +/* The following are only used by the kernel +* and are not part of the uapi +*/ +struct { +__aligned_u64 addr; +__aligned_u64 length; +union { +struct ib_mr *mr; +__aligned_u64 reserved1; + }; +union { +struct ib_mw *mw; +__aligned_u64 reserved2; + }; +__u32 rkey; +__u32 access; +__u32 flags; + } kmw; struct { union { struct ib_mr *mr;
Signature · 1 line
2.25.1
PATCH v7 8/19 rdma_rxe: Add memory access through MWs
In reply to Bob Pearsonv7:
- adjust for pool API changes
v6:
- Implement memory access through MWs.
- Add rules checks from IBA.
--- drivers/infiniband/sw/rxe/rxe_loc.h | 19 +++--- drivers/infiniband/sw/rxe/rxe_mr.c | 76 +++++++++++++++--------- drivers/infiniband/sw/rxe/rxe_mw.c | 57 +++++++++++++++--- drivers/infiniband/sw/rxe/rxe_pool.c | 14 +++-- drivers/infiniband/sw/rxe/rxe_pool.h | 3 +- drivers/infiniband/sw/rxe/rxe_req.c | 15 ++--- drivers/infiniband/sw/rxe/rxe_resp.c | 83 ++++++++++++++++++++------- drivers/infiniband/sw/rxe/rxe_verbs.h | 1 + 8 files changed, 189 insertions(+), 79 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +10 -9
@@ -100,28 +100,29 @@ enum lookup_type { lookup_remote, }; -struct rxe_mr *lookup_mr (struct rxe_pd *pd,int access,u32 key, -enum lookup_type type); - -int mr_check_range (struct rxe_mr *mr,u64 iova,size_t length); - -void rxe_mr_cleanup (struct rxe_pool_entry *arg); -int advance_dma_data (struct rxe_dma_info *dma,unsigned int length);int rxe_invalidate_mr (struct rxe_qp *qp,struct rxe_mr *mr); +int rxe_mr_check_access (struct rxe_qp *qp,struct rxe_mr *mr, +int access,u64 va,u32 resid); + +void rxe_mr_cleanup (struct rxe_pool_entry *arg); +/* rxe_mw.c */ int rxe_alloc_mw (struct ib_mw *ibmw,struct ib_udata *udata);int rxe_dealloc_mw (struct ib_mw *ibmw); -void rxe_mw_cleanup (struct rxe_pool_entry *arg); -int rxe_bind_mw (struct rxe_qp *qp,struct rxe_send_wqe *wqe);int rxe_invalidate_mw (struct rxe_qp *qp,struct rxe_mw *mw); +int rxe_mw_check_access (struct rxe_qp *qp,struct rxe_mw *mw, +int access,u64 va,u32 resid); + +void rxe_mw_cleanup (struct rxe_pool_entry *arg); +/* rxe_net.c */ void rxe_loopback (struct sk_buff *skb);int rxe_send (struct rxe_pkt_info *pkt,struct sk_buff *skb);
drivers/infiniband/sw/rxe/rxe_mr.c +50 -26
112 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_mw.c +49 -8
75 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_pool.c +10 -4
@@ -72,6 +72,7 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = { .name = "rxe-mw", .size = sizeof(struct rxe_mw), .elem_offset = offsetof(struct rxe_mw, pelem), + .cleanup = rxe_mw_cleanup, .flags =RXE_POOL_NO_ALLOC |RXE_POOL_INDEX | RXE_POOL_KEY,
@@ -220,11 +221,12 @@ static u32 alloc_index(struct rxe_pool *pool)return index + pool->index.min_index; } -static void insert_index (struct rxe_pool *pool,struct rxe_pool_entry *new) +static int insert_index (struct rxe_pool *pool,struct rxe_pool_entry *new) {struct rb_node **link = &pool->index.tree.rb_node;struct rb_node *parent =NULL ;struct rxe_pool_entry *elem; +int ret =0 ;while (*link) { parent = *link;
@@ -232,6 +234,7 @@ static void insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new)if (elem->index == new->index) {pr_warn ("element already exists!\n" ); + ret = -EINVAL ; goto out; }
@@ -244,7 +247,7 @@ static void insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new)rb_link_node (&new->index_node, parent, link);rb_insert_color (&new->index_node, &pool->index.tree); out: -return ; +return ret; }static int insert_key (struct rxe_pool *pool,struct rxe_pool_entry *new)
@@ -302,15 +305,18 @@ void __rxe_drop_key(struct rxe_pool_entry *elem)write_unlock_irqrestore (&pool->pool_lock, flags); } -void __rxe_add_index (struct rxe_pool_entry *elem) +int __rxe_add_index (struct rxe_pool_entry *elem) {struct rxe_pool *pool = elem->pool;unsigned long flags; +int ret;write_lock_irqsave (&pool->pool_lock, flags); elem->index =alloc_index (pool); -insert_index (pool, elem); + ret =insert_index (pool, elem);write_unlock_irqrestore (&pool->pool_lock, flags); + +return ret; }void __rxe_drop_index (struct rxe_pool_entry *elem)
drivers/infiniband/sw/rxe/rxe_pool.h +2 -1
@@ -113,7 +113,7 @@ int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_entry *elem);#define rxe_add_to_pool (pool, obj) __rxe_add_to_pool(pool, &(obj)->pelem) -void __rxe_add_index (struct rxe_pool_entry *elem); +int __rxe_add_index (struct rxe_pool_entry *elem);#define rxe_add_index (obj) __rxe_add_index(&(obj)->pelem)
@@ -144,4 +144,5 @@ void rxe_elem_release(struct kref *kref);/* drop a reference on an object */ #define rxe_drop_ref (elem) kref_put(&(elem)->pelem.ref_cnt, rxe_elem_release) +#define rxe_read_ref (elem) kref_read(&(elem)->pelem.ref_cnt)#endif /* RXE_POOL_H */
drivers/infiniband/sw/rxe/rxe_req.c +5 -10
@@ -645,7 +645,6 @@ int rxe_requester(void *arg)case IB_WR_BIND_MW : ret =rxe_bind_mw (qp, wqe);if (ret) { - wqe->state = wqe_state_done; wqe->status =IB_WC_MW_BIND_ERR ; goto err; }
@@ -653,6 +652,7 @@ int rxe_requester(void *arg)default :pr_err_once ("unexpected LOCAL WR opcode = %d\n" , wqe->wr.opcode); + wqe->status =IB_WC_LOC_QP_OP_ERR ; goto err; }
@@ -698,13 +698,7 @@ int rxe_requester(void *arg) payload = (mask &RXE_WRITE_OR_SEND ) ? wqe->dma.resid :0 ;if (payload > mtu) {if (qp_type (qp) ==IB_QPT_UD ) { -/* C10-93.1.1: If the total sum of all the buffer lengths specified for a -* UD message exceeds the MTU of the port as returned by QueryHCA, the CI -* shall not emit any packets for this message. Further, the CI shall not -* generate an error due to this condition. -*/ - -/* fake a successful UD send */ +/* C10-93.1.1: fake a successful UD send */ wqe->first_psn = qp->req.psn; wqe->last_psn = qp->req.psn; qp->req.psn = (qp->req.psn +1 ) &BTH_PSN_MASK ;
@@ -769,6 +763,8 @@ int rxe_requester(void *arg)* to be called again */ wqe->state = wqe_state_error; + qp->req.wqe_index =next_index (qp->sq.queue, + qp->req.wqe_index);__rxe_do_task (&qp->comp.task); ret = -EAGAIN ; goto done;
@@ -784,8 +780,7 @@ int rxe_requester(void *arg) again:/* we come here if we are done with the current wqe but want to -* get called again. Mostly we loop back to next wqe so should -* be all one way or the other +* get called again. */ ret =0 ; goto done;
drivers/infiniband/sw/rxe/rxe_resp.c +62 -21
157 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_verbs.h +1 -0
@@ -183,6 +183,7 @@ struct rxe_resp_info {/* RDMA read / atomic only */ u64 va; +u64 offset;struct rxe_mr *mr;u32 resid;u32 rkey;
Signature · 1 line
2.25.1
PATCH v7 9/19 rdma_rxe: Add locked and unlocked pool APIs
In reply to Bob Pearsonv7:
- Current object pool APIs use the pool_lock to protect concurrent execution from race conditions. Most APIs take the pool_lock, execute one operation and then free the lock.
This patch adds unlocked versions of the APIs so that a caller can take the pool_lock, combine sequences of operations and then free the lock. This will be required to fix some race conditions which occur without this fix.
--- drivers/infiniband/sw/rxe/rxe_pool.c | 528 ++++++++++++++++++++------- drivers/infiniband/sw/rxe/rxe_pool.h | 111 +++--- 2 files changed, 467 insertions(+), 172 deletions(-)
drivers/infiniband/sw/rxe/rxe_pool.c +400 -128
@@ -7,9 +7,8 @@#include "rxe.h" #include "rxe_loc.h" -/* info about object pools -*/ -struct rxe_type_info rxe_type_info[RXE_NUM_TYPES ] = { +/* info about object pools */ +static struct rxe_type_info rxe_type_info[RXE_NUM_TYPES ] = { [RXE_TYPE_UC ] = { .name ="rxe-uc" , .size =sizeof (struct rxe_ucontext ),
@@ -62,8 +61,8 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = { .flags = RXE_POOL_ATOMIC |RXE_POOL_INDEX | RXE_POOL_KEY, - .max_index = RXE_MAX_MR_INDEX, .min_index = RXE_MIN_MR_INDEX, + .max_index = RXE_MAX_MR_INDEX, .key_offset = offsetof(struct rxe_mr, ibmr.lkey) - offsetof(struct rxe_mr, pelem), .key_size = sizeof(u32),
@@ -76,8 +75,8 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = { .flags = RXE_POOL_NO_ALLOC |RXE_POOL_INDEX | RXE_POOL_KEY, - .max_index = RXE_MAX_MW_INDEX, .min_index = RXE_MIN_MW_INDEX, + .max_index = RXE_MAX_MW_INDEX, .key_offset = offsetof(struct rxe_mw, ibmw.rkey) - offsetof(struct rxe_mw, pelem), .key_size = sizeof(u32),
@@ -106,21 +105,18 @@ static inline const char *pool_name(struct rxe_pool *pool)return rxe_type_info[pool->type].name; } -static int rxe_pool_init_index (struct rxe_pool *pool,u32 max,u32 min) +static int init_index_table (struct rxe_pool *pool,u32 indices) {int err =0 ;size_t size; -if ((max - min +1 ) < pool->max_elem) { +if (indices < pool->max_elem) {pr_warn ("not enough indices for max_elem\n" ); err = -EINVAL ; goto out; } - pool->index.max_index = max; - pool->index.min_index = min; - - size =BITS_TO_LONGS (max - min +1 ) *sizeof (long ); + size =BITS_TO_LONGS (indices) *sizeof (long ); pool->index.table =kmalloc (size,GFP_KERNEL );if (!pool->index.table) { err = -ENOMEM ;
@@ -128,31 +124,37 @@ static int rxe_pool_init_index(struct rxe_pool *pool, u32 max, u32 min) } pool->index.table_size = size; -bitmap_zero (pool->index.table, max - min +1 ); - +bitmap_zero (pool->index.table, indices); out:return err; } -int rxe_pool_init ( -struct rxe_dev *rxe, -struct rxe_pool *pool, -enum rxe_elem_type type, -unsigned int max_elem) +/** +* rxe_pool_init - Initialize an object pool +* @rxe: The rxe device containing pool +* @pool: The pool to initialize +* @type: The object type held in pool +* @max_elem: The pool size in elements +* +* Initialize a pool of objects with given limit on +* number of elements. gets parameters from rxe_type_info +*/ +int rxe_pool_init (struct rxe_dev *rxe,struct rxe_pool *pool, +enum rxe_elem_type type,unsigned int max_elem) { -int err =0 ; -size_t size = rxe_type_info[type].size; +struct rxe_type_info *info = &rxe_type_info[type]; +int err =0 ;memset (pool,0 ,sizeof (*pool)); pool->rxe = rxe; pool->type = type; pool->max_elem = max_elem; - pool->elem_size =ALIGN (size,RXE_POOL_ALIGN ); - pool->flags = rxe_type_info[type].flags; + pool->elem_size =ALIGN (info->size,RXE_POOL_ALIGN ); + pool->flags = info->flags; pool->index.tree =RB_ROOT ; pool->key.tree =RB_ROOT ; - pool->cleanup = rxe_type_info[type].cleanup; + pool->cleanup = info->cleanup;atomic_set (&pool->num_elem,0 );
@@ -160,26 +162,27 @@ int rxe_pool_init(rwlock_init (&pool->pool_lock); -if (rxe_type_info[type].flags &RXE_POOL_INDEX ) { - err =rxe_pool_init_index (pool, - rxe_type_info[type].max_index, - rxe_type_info[type].min_index); +if (info->flags &RXE_POOL_INDEX ) { + pool->index.min_index = info->min_index; + pool->index.max_index = info->max_index; + + err =init_index_table (pool, info->max_index + - info->min_index +1 );if (err) goto out; } -if (rxe_type_info[type].flags &RXE_POOL_KEY ) { - pool->key.key_offset = rxe_type_info[type].key_offset; - pool->key.key_size = rxe_type_info[type].key_size; +if (info->flags &RXE_POOL_KEY ) { + pool->key.key_offset = info->key_offset; + pool->key.key_size = info->key_size; } pool->state =RXE_POOL_STATE_VALID ; - out:return err; } -static void rxe_pool_release (struct kref *kref) +static void pool_release (struct kref *kref) { struct rxe_pool *pool = container_of(kref, struct rxe_pool, ref_cnt);
@@ -187,11 +190,19 @@ static void rxe_pool_release(struct kref *kref)kfree (pool->index.table); } -static void rxe_pool_put (struct rxe_pool *pool) -{ -kref_put (&pool->ref_cnt, rxe_pool_release); -} - +#define pool_get (pool) kref_get(&(pool)->ref_cnt) +#define pool_put (pool) kref_put(&(pool)->ref_cnt, pool_release) + +/** +* rxe_pool_cleanup - Cleanup an object pool +* @pool: The pool to cleanup +* +* Marks pool as invalid and drops a ref to the pool. +* When the ref count gets to zero pool_release will +* delete the table memory held by the pool. +* Pool entries hold references to the pool so it will +* hang around until they are all removed. +*/ void rxe_pool_cleanup (struct rxe_pool *pool) {unsigned long flags;
@@ -203,9 +214,15 @@ void rxe_pool_cleanup(struct rxe_pool *pool) pool_name(pool));write_unlock_irqrestore (&pool->pool_lock, flags); -rxe_pool_put (pool); +pool_put (pool); } +/* +* Allocate a new index in the pool from the index bit table. +* Search from the last allocated index forward. +* Caller should hold pool->pool_lock to be thread safe. +* There should be more possible indices than objects in the pool. +*/ static u32 alloc_index (struct rxe_pool *pool) {u32 index;
@@ -215,12 +232,21 @@ static u32 alloc_index(struct rxe_pool *pool)if (index >= range) index =find_first_zero_bit (pool->index.table, range); -WARN_ON_ONCE (index >= range); +if (unlikely (index >= range)) +pr_err ("Unable to allocate a new index in %s pool\n" , +pool_name (pool)); +set_bit (index, pool->index.table); pool->index.last = index; +return index + pool->index.min_index; } +/* +* Insert index in index-red-black tree in pool. +* Used to lookup object by index. Returns an error +* if index is already inserted. +*/ static int insert_index (struct rxe_pool *pool,struct rxe_pool_entry *new) {struct rb_node **link = &pool->index.tree.rb_node;
@@ -232,8 +258,9 @@ static int insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new) parent = *link; elem = rb_entry(parent, struct rxe_pool_entry, index_node); -if (elem->index == new->index) { -pr_warn ("element already exists!\n" ); +if (unlikely (elem->index == new->index)) { +pr_warn ("index %d already exists in %s pool\n" , + new->index,pool_name (pool)); ret = -EINVAL ; goto out; }
@@ -250,11 +277,79 @@ static int insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new)return ret; } +/** +* __add_index - Add index to pool entry +* @elem: pool entry to add index to +* +* Allocate a new index and insert into index-red-black tree. +* Caller should hold pool->pool_lock. +*/ +void __add_index (struct rxe_pool_entry *elem) +{ +struct rxe_pool *pool = elem->pool; + + elem->index =alloc_index (pool); +insert_index (pool, elem); +} + +/** +* __rxe_add_index - Lock and add index to pool entry +* @elem: pool entry to add index to +* +* Allocate a new index and insert into index-red-black tree. +*/ +void __rxe_add_index (struct rxe_pool_entry *elem) +{ +struct rxe_pool *pool = elem->pool; +unsigned long flags; + +write_lock_irqsave (&pool->pool_lock, flags); +__add_index (elem); +write_unlock_irqrestore (&pool->pool_lock, flags); +} + +/** +* __drop_index - Remove index from pool entry +* @elem: pool entry to remove index from +* +* Remove index from bit table and delete entry from index-red-black tree. +* Caller should hold pool->pool_lock. +*/ +void __drop_index (struct rxe_pool_entry *elem) +{ +struct rxe_pool *pool = elem->pool; + +clear_bit (elem->index - pool->index.min_index, pool->index.table); +rb_erase (&elem->index_node, &pool->index.tree); +} + +/** +* __rxe_drop_index - Lock and remove index from pool entry +* @elem: pool entry to remove index from +* +* Remove index from bit table and delete from index-red-black tree. +*/ +void __rxe_drop_index (struct rxe_pool_entry *elem) +{ +struct rxe_pool *pool = elem->pool; +unsigned long flags; + +write_lock_irqsave (&pool->pool_lock, flags); +__drop_index (elem); +write_unlock_irqrestore (&pool->pool_lock, flags); +} + +/* +* Insert key in key red-black tree in pool. +* Used to lookup object by key. Returns an error +* if key is already inserted. +*/ static int insert_key (struct rxe_pool *pool,struct rxe_pool_entry *new) {struct rb_node **link = &pool->key.tree.rb_node;struct rb_node *parent =NULL ;struct rxe_pool_entry *elem; +int ret =0 ;int cmp;while (*link) {
@@ -262,11 +357,14 @@ static int insert_key(struct rxe_pool *pool, struct rxe_pool_entry *new) elem = rb_entry(parent, struct rxe_pool_entry, key_node); cmp =memcmp ((u8 *)elem + pool->key.key_offset, - (u8 *)new + pool->key.key_offset, pool->key.key_size); + (u8 *)new + pool->key.key_offset, + pool->key.key_size); -if (cmp ==0 ) { -pr_warn ("key already exists!\n" ); -return -EAGAIN ; +if (unlikely (cmp ==0 )) { +pr_warn ("key already exists in %s pool\n" , +pool_name (pool)); + ret = -EAGAIN ; + goto out; }if (cmp >0 )
@@ -277,75 +375,98 @@ static int insert_key(struct rxe_pool *pool, struct rxe_pool_entry *new)rb_link_node (&new->key_node, parent, link);rb_insert_color (&new->key_node, &pool->key.tree); - -return 0 ; +out: +return ret; } -int __rxe_add_key (struct rxe_pool_entry *elem,void *key) +/** +* __add_key - Add key to pool entry +* @elem: pool entry to add index to +* @key: the key to add +* +* Copy key into entry and insert into key-red-black tree. +* Caller should hold pool->pool_lock. +*/ +int __add_key (struct rxe_pool_entry *elem,void *key) { -struct rxe_pool *pool = elem->pool; -unsigned long flags;int ret; +struct rxe_pool *pool = elem->pool; -write_lock_irqsave (&pool->pool_lock, flags);memcpy ((u8 *)elem + pool->key.key_offset, key, pool->key.key_size); ret =insert_key (pool, elem); -write_unlock_irqrestore (&pool->pool_lock, flags);return ret; } -void __rxe_drop_key (struct rxe_pool_entry *elem) +/** +* __rxe_add_key - Lock and add key to pool entry +* @elem: pool entry to add index to +* @key: the key to add +* +* Copy key into entry and insert into key-red-black tree. +*/ +int __rxe_add_key (struct rxe_pool_entry *elem,void *key) { +int ret;struct rxe_pool *pool = elem->pool;unsigned long flags;write_lock_irqsave (&pool->pool_lock, flags); -rb_erase (&elem->key_node, &pool->key.tree); + ret =__add_key (elem, key);write_unlock_irqrestore (&pool->pool_lock, flags); + +return ret; } -int __rxe_add_index (struct rxe_pool_entry *elem) +/** +* __drop_key - Remove key from pool entry +* @elem: pool entry to remove key from +* +* Delete entry from key-red-black tree. +* Caller should hold pool->pool_lock. +*/ +void __drop_key (struct rxe_pool_entry *elem) {struct rxe_pool *pool = elem->pool; -unsigned long flags; -int ret; - -write_lock_irqsave (&pool->pool_lock, flags); - elem->index =alloc_index (pool); - ret =insert_index (pool, elem); -write_unlock_irqrestore (&pool->pool_lock, flags); -return ret; +rb_erase (&elem->key_node, &pool->key.tree); } -void __rxe_drop_index (struct rxe_pool_entry *elem) +/** +* __rxe_drop_key - Lock and remove key from pool entry +* @elem: pool entry to remove key from +* +* Delete entry from key-red-black tree. +*/ +void __rxe_drop_key (struct rxe_pool_entry *elem) {struct rxe_pool *pool = elem->pool;unsigned long flags;write_lock_irqsave (&pool->pool_lock, flags); -clear_bit (elem->index - pool->index.min_index, pool->index.table); -rb_erase (&elem->index_node, &pool->index.tree); +__drop_key (elem);write_unlock_irqrestore (&pool->pool_lock, flags); } -void *rxe_alloc (struct rxe_pool *pool) +/** +* __add_to_pool - Add pool entry allocated elsewhere to pool +* @pool: Pool to add entry to +* @elem: Pool entry to add +* +* Used to add objects allocated in rdma-core to the pool. +* Entry holds a reference to pool to protect the pool pointer +* Takes a reference to the ib_device so it can't be deleted +* before the pool is deleted. +* Returns an error if pool has been deleted or if there are +* too many objects in the pool. +* Caller should hold the pool->pool_lock. +*/ +int __add_to_pool (struct rxe_pool *pool,struct rxe_pool_entry *elem) { -struct rxe_pool_entry *elem; -struct rxe_type_info *info = &rxe_type_info[pool->type]; -u8 *obj =NULL ; -unsigned long flags; +if (pool->state !=RXE_POOL_STATE_VALID ) +return -EINVAL ; -might_sleep_if (!(pool->flags &RXE_POOL_ATOMIC )); - -read_lock_irqsave (&pool->pool_lock, flags); -if (pool->state !=RXE_POOL_STATE_VALID ) { -read_unlock_irqrestore (&pool->pool_lock, flags); -return NULL ; - } -kref_get (&pool->ref_cnt); -read_unlock_irqrestore (&pool->pool_lock, flags); +pool_get (pool);if (!ib_device_try_get (&pool->rxe->ib_dev)) goto out_put_pool;
@@ -353,85 +474,153 @@ void *rxe_alloc(struct rxe_pool *pool)if (atomic_inc_return (&pool->num_elem) > pool->max_elem) goto out_cnt; - elem =kzalloc (rxe_type_info[pool->type].size, - (pool->flags &RXE_POOL_ATOMIC ) ? -GFP_ATOMIC :GFP_KERNEL ); -if (!elem) - goto out_cnt; - elem->pool = pool;kref_init (&elem->ref_cnt); - obj = (u8 *)elem - info->elem_offset; - -return obj; +return 0 ; out_cnt:atomic_dec (&pool->num_elem);ib_device_put (&pool->rxe->ib_dev); out_put_pool: -rxe_pool_put (pool); -return NULL ; +pool_put (pool); +return -EINVAL ; } +/** +* __rxe_add_to_pool - Lock and add pool entry allocated elsewhere to pool +* @pool: Pool to add entry to +* @elem: Pool entry to add +* +* Same as __add_to_pool except takes the pool_lock. +*/ int __rxe_add_to_pool (struct rxe_pool *pool,struct rxe_pool_entry *elem) { +int err;unsigned long flags; -read_lock_irqsave (&pool->pool_lock, flags); +write_lock_irqsave (&pool->pool_lock, flags); + err =__add_to_pool (pool, elem); +write_unlock_irqrestore (&pool->pool_lock, flags); + +return err; +} + +/** +* __alloc - Create a new pool entry +* @pool: Pool to create entry in +* +* Entry holds a reference to pool to protect the pool pointer +* Takes a reference to the ib_device so it can't be deleted +* before the pool is deleted. +* Returns an error pointer if pool has been deleted or if there are +* too many objects in the pool or it cannot get memory for the +* object. +* Caller should hold the pool->pool_lock. +*/ +void *__alloc (struct rxe_pool *pool) +{ +struct rxe_type_info *info = &rxe_type_info[pool->type]; +void *obj; +struct rxe_pool_entry *elem; +int ret =0 ; +if (pool->state !=RXE_POOL_STATE_VALID ) { -read_unlock_irqrestore (&pool->pool_lock, flags); -return -EINVAL ; + ret = -EINVAL ; + goto out; } -kref_get (&pool->ref_cnt); -read_unlock_irqrestore (&pool->pool_lock, flags); -if (!ib_device_try_get (&pool->rxe->ib_dev)) +pool_get (pool); + +if (!ib_device_try_get (&pool->rxe->ib_dev)) { + ret = -EINVAL ; goto out_put_pool; + } -if (atomic_inc_return (&pool->num_elem) > pool->max_elem) +if (atomic_inc_return (&pool->num_elem) > pool->max_elem) { + ret = -EINVAL ; + goto out_cnt; + } + + obj =kzalloc (info->size,GFP_ATOMIC ); +if (!obj) { + ret = -ENOMEM ; goto out_cnt; + } + + elem = (struct rxe_pool_entry *)((u8 *)obj + info->elem_offset); elem->pool = pool;kref_init (&elem->ref_cnt); -return 0 ; +return obj; out_cnt:atomic_dec (&pool->num_elem);ib_device_put (&pool->rxe->ib_dev); out_put_pool: -rxe_pool_put (pool); -return -EINVAL ; +pool_put (pool); +out: +return ERR_PTR (ret); } -void rxe_elem_release (struct kref *kref) +/** +* rxe_alloc - Lock and create a new pool entry +* @pool: Pool to create entry in +* +* Same as __alloc except holds the pool_lock. +*/ +void *rxe_alloc (struct rxe_pool *pool) { -struct rxe_pool_entry *elem = - container_of(kref, struct rxe_pool_entry, ref_cnt); -struct rxe_pool *pool = elem->pool; +void *obj; +struct rxe_pool_entry *elem; +unsigned long flags; +int err; -if (pool->cleanup) - pool->cleanup (elem); +/* allocate object outside of lock in case it sleeps */ + obj =kzalloc (rxe_type_info[pool->type].size, + (pool->flags &RXE_POOL_ATOMIC ) ? +GFP_ATOMIC :GFP_KERNEL ); +if (!obj) + goto out; -if (!(pool->flags &RXE_POOL_NO_ALLOC )) -kfree (elem); -atomic_dec (&pool->num_elem); -ib_device_put (&pool->rxe->ib_dev); -rxe_pool_put (pool); + elem = (struct rxe_pool_entry *)((u8 *)obj + + rxe_type_info[pool->type].elem_offset); + +write_lock_irqsave (&pool->pool_lock, flags); + err =__add_to_pool (pool, elem); +write_unlock_irqrestore (&pool->pool_lock, flags); +if (err) { +kfree (obj); + obj =NULL ; + } +out: +return obj; } -void *rxe_get_index (struct rxe_pool *pool,u32 index) +/** +* __get_index - Lookup index in pool and return object +* @pool: pool to lookup index in +* @index: index to lookup +* +* Lookup an index in pool's index-red-black tree and retrieve +* pool entry with that index. +* Convert entry address to object address with elem_offset. +* Returns the object address if found else NULL if not or +* an error pointer if the pool has been destroyed. +* Caller should hold the pool->pool_lock. +*/ +void *__get_index (struct rxe_pool *pool,u32 index) { -struct rb_node *node =NULL ; -struct rxe_pool_entry *elem =NULL ;struct rxe_type_info *info = &rxe_type_info[pool->type]; -u8 *obj =NULL ; -unsigned long flags; - -read_lock_irqsave (&pool->pool_lock, flags); +struct rb_node *node; +struct rxe_pool_entry *elem =NULL ; +void *obj; +int ret =0 ; -if (pool->state !=RXE_POOL_STATE_VALID ) +if (pool->state !=RXE_POOL_STATE_VALID ) { + ret = -EINVAL ; goto out; + } node = pool->index.tree.rb_node;
@@ -449,25 +638,59 @@ void *rxe_get_index(struct rxe_pool *pool, u32 index)if (node) {kref_get (&elem->ref_cnt); obj = (u8 *)elem - info->elem_offset; + }else { + obj =NULL ; } + +return obj; out: +return ERR_PTR (ret); +} + +/** +* rxe_get_index - Lock and lookup index in pool and return object +* @pool: pool to lookup index in +* @index: index to lookup +* +* Same as __get_index except holds pool_lock. +*/ +void *rxe_get_index (struct rxe_pool *pool,u32 index) +{ +void *obj; +unsigned long flags; + +read_lock_irqsave (&pool->pool_lock, flags); + obj =__get_index (pool, index);read_unlock_irqrestore (&pool->pool_lock, flags); +return obj; } -void *rxe_get_key (struct rxe_pool *pool,void *key) +/** +* __get_key - Lookup key in pool and return object +* @pool: pool to lookup key in +* @key: key to lookup +* +* Lookup a key in pool's key-red-black tree and retrieve +* pool entry with that key. +* Convert entry address to object address with elem_offset. +* Returns the object address if found else NULL if not or +* an error pointer if the pool has been destroyed. +* Caller should hold the pool->pool_lock. +*/ +void *__get_key (struct rxe_pool *pool,void *key) { -struct rb_node *node =NULL ; -struct rxe_pool_entry *elem =NULL ;struct rxe_type_info *info = &rxe_type_info[pool->type]; -u8 *obj =NULL ; +struct rb_node *node; +struct rxe_pool_entry *elem =NULL ;int cmp; -unsigned long flags; - -read_lock_irqsave (&pool->pool_lock, flags); +void *obj; +int ret =0 ; -if (pool->state !=RXE_POOL_STATE_VALID ) +if (pool->state !=RXE_POOL_STATE_VALID ) { + ret = -EINVAL ; goto out; + } node = pool->key.tree.rb_node;
@@ -488,9 +711,58 @@ void *rxe_get_key(struct rxe_pool *pool, void *key)if (node) {kref_get (&elem->ref_cnt); obj = (u8 *)elem - info->elem_offset; + }else { + obj =NULL ; } +return obj; out: +return ERR_PTR (ret); +} + +/** +* rxe_get_key - Lock and lookup key in pool and return object +* @pool: pool to lookup key in +* @key: key to lookup +* +* Same as __get_key except holds pool_lock. +*/ +void *rxe_get_key (struct rxe_pool *pool,void *key) +{ +void *obj; +unsigned long flags; + +read_lock_irqsave (&pool->pool_lock, flags); + obj =__get_key (pool, key);read_unlock_irqrestore (&pool->pool_lock, flags); +return obj; } + +/** +* rxe_elem_release - Delete pool object when ref count goes to zero +* @kref: Reference counter contained in pool entry +* +* If pool has an object cleanup routine call it. +* If entry memory was allocated in pool free it. +*/ +void rxe_elem_release (struct kref *kref) +{ +struct rxe_pool_entry *elem =container_of (kref, + struct rxe_pool_entry, ref_cnt); +struct rxe_pool *pool = elem->pool; +struct rxe_type_info *info = &rxe_type_info[pool->type]; +void *obj; + +if (pool->cleanup) + pool->cleanup (elem); + +if (!(pool->flags &RXE_POOL_NO_ALLOC )) { + obj = (u8 *)elem - info->elem_offset; +kfree (obj); + } + +atomic_dec (&pool->num_elem); +ib_device_put (&pool->rxe->ib_dev); +pool_put (pool); +}
drivers/infiniband/sw/rxe/rxe_pool.h +67 -44
184 lines not on this page · raw message
Signature · 1 line
2.25.1
PATCH v7 10/19 rdma_rxe: Add support for ibv_query_device_ex
In reply to Bob PearsonAdd code to initialize new struct members inib_device_attr as place holders.
--- drivers/infiniband/sw/rxe/rxe.c | 101 ++++++++++++++++++-------- drivers/infiniband/sw/rxe/rxe_verbs.c | 7 +- 2 files changed, 75 insertions(+), 33 deletions(-)
drivers/infiniband/sw/rxe/rxe.c +69 -32
@@ -40,40 +40,77 @@ void rxe_dealloc(struct ib_device *ib_dev)/* initialize rxe device parameters */ static void rxe_init_device_param (struct rxe_dev *rxe) { - rxe->max_inline_data =RXE_MAX_INLINE_DATA ; - - rxe->attr.vendor_id =RXE_VENDOR_ID ; - rxe->attr.max_mr_size =RXE_MAX_MR_SIZE ; - rxe->attr.page_size_cap =RXE_PAGE_SIZE_CAP ; - rxe->attr.max_qp =RXE_MAX_QP ; - rxe->attr.max_qp_wr =RXE_MAX_QP_WR ; - rxe->attr.device_cap_flags =RXE_DEVICE_CAP_FLAGS ; - rxe->attr.max_send_sge =RXE_MAX_SGE ; - rxe->attr.max_recv_sge =RXE_MAX_SGE ; - rxe->attr.max_sge_rd =RXE_MAX_SGE_RD ; - rxe->attr.max_cq =RXE_MAX_CQ ; - rxe->attr.max_cqe = (1 <<RXE_MAX_LOG_CQE ) -1 ; - rxe->attr.max_mr =RXE_MAX_MR ; - rxe->attr.max_mw =RXE_MAX_MW ; - rxe->attr.max_pd =RXE_MAX_PD ; - rxe->attr.max_qp_rd_atom =RXE_MAX_QP_RD_ATOM ; - rxe->attr.max_res_rd_atom =RXE_MAX_RES_RD_ATOM ; - rxe->attr.max_qp_init_rd_atom =RXE_MAX_QP_INIT_RD_ATOM ; - rxe->attr.atomic_cap =IB_ATOMIC_HCA ; - rxe->attr.max_mcast_grp =RXE_MAX_MCAST_GRP ; - rxe->attr.max_mcast_qp_attach =RXE_MAX_MCAST_QP_ATTACH ; - rxe->attr.max_total_mcast_qp_attach =RXE_MAX_TOT_MCAST_QP_ATTACH ; - rxe->attr.max_ah =RXE_MAX_AH ; - rxe->attr.max_srq =RXE_MAX_SRQ ; - rxe->attr.max_srq_wr =RXE_MAX_SRQ_WR ; - rxe->attr.max_srq_sge =RXE_MAX_SRQ_SGE ; - rxe->attr.max_fast_reg_page_list_len =RXE_MAX_FMR_PAGE_LIST_LEN ; - rxe->attr.max_pkeys =RXE_MAX_PKEYS ; - rxe->attr.local_ca_ack_delay =RXE_LOCAL_CA_ACK_DELAY ; -addrconf_addr_eui48 ((unsigned char *)&rxe->attr.sys_image_guid, - rxe->ndev->dev_addr); +struct ib_device_attr *a = &rxe->attr; + rxe->max_inline_data =RXE_MAX_INLINE_DATA ; rxe->max_ucontext =RXE_MAX_UCONTEXT ; + + a->atomic_cap =IB_ATOMIC_HCA ; + a->cq_caps.max_cq_moderation_count =0 ; + a->cq_caps.max_cq_moderation_period =0 ; + a->device_cap_flags =RXE_DEVICE_CAP_FLAGS ; + a->fw_ver =0 ; + a->hca_core_clock =0 ; + a->hw_ver =0 ; + a->local_ca_ack_delay =RXE_LOCAL_CA_ACK_DELAY ; + a->masked_atomic_cap =0 ; + a->max_ah =RXE_MAX_AH ; + a->max_cqe = (1 <<RXE_MAX_LOG_CQE ) -1 ; + a->max_cq =RXE_MAX_CQ ; + a->max_dm_size =0 ; + a->max_ee_init_rd_atom =0 ; + a->max_ee =0 ; + a->max_ee_rd_atom =0 ; + a->max_fast_reg_page_list_len =RXE_MAX_FMR_PAGE_LIST_LEN ; + a->max_mcast_grp =RXE_MAX_MCAST_GRP ; + a->max_mcast_qp_attach =RXE_MAX_MCAST_QP_ATTACH ; + a->max_mr =RXE_MAX_MR ; + a->max_mr_size =RXE_MAX_MR_SIZE ; + a->max_mw =RXE_MAX_MW ; + a->max_pd =RXE_MAX_PD ; + a->max_pi_fast_reg_page_list_len =0 ; + a->max_pkeys =RXE_MAX_PKEYS ; + a->max_qp_init_rd_atom =RXE_MAX_QP_INIT_RD_ATOM ; + a->max_qp_rd_atom =RXE_MAX_QP_RD_ATOM ; + a->max_qp =RXE_MAX_QP ; + a->max_qp_wr =RXE_MAX_QP_WR ; + a->max_raw_ethy_qp =0 ; + a->max_raw_ipv6_qp =0 ; + a->max_rdd =0 ; + a->max_recv_sge =RXE_MAX_SGE ; + a->max_res_rd_atom =RXE_MAX_RES_RD_ATOM ; + a->max_send_sge =RXE_MAX_SGE ; + a->max_sge_rd =RXE_MAX_SGE_RD ; + a->max_sgl_rd =0 ; + a->max_srq =RXE_MAX_SRQ ; + a->max_srq_sge =RXE_MAX_SRQ_SGE ; + a->max_srq_wr =RXE_MAX_SRQ_WR ; + a->max_total_mcast_qp_attach =RXE_MAX_TOT_MCAST_QP_ATTACH ; + a->max_wq_type_rq =0 ; + a->odp_caps.general_caps =0 ; + a->odp_caps.per_transport_caps.rc_odp_caps =0 ; + a->odp_caps.per_transport_caps.uc_odp_caps =0 ; + a->odp_caps.per_transport_caps.ud_odp_caps =0 ; + a->odp_caps.per_transport_caps.xrc_odp_caps =0 ; + a->page_size_cap =RXE_PAGE_SIZE_CAP ; + a->raw_packet_caps =0 ; + a->rss_caps.supported_qpts =0 ; + a->rss_caps.max_rwq_indirection_tables =0 ; + a->rss_caps.max_rwq_indirection_table_size =0 ; + a->sig_guard_cap =0 ; + a->sig_prot_cap =0 ; + a->sys_image_guid =0 ; + a->timestamp_mask =0 ; + a->tm_caps.max_rndv_hdr_size =0 ; + a->tm_caps.max_num_tags =0 ; + a->tm_caps.flags =0 ; + a->tm_caps.max_ops =0 ; + a->tm_caps.max_sge =0 ; + a->vendor_id =RXE_VENDOR_ID ; + a->vendor_part_id =0 ; + +addrconf_addr_eui48 ((unsigned char *)&a->sys_image_guid, + rxe->ndev->dev_addr); }/* initialize port attributes */
drivers/infiniband/sw/rxe/rxe_verbs.c +6 -1
@@ -1150,7 +1150,8 @@ int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name)dma_coerce_mask_and_coherent (&dev->dev,dma_get_required_mask (&dev->dev)); - dev->uverbs_cmd_mask =BIT_ULL (IB_USER_VERBS_CMD_GET_CONTEXT ) + dev->uverbs_cmd_mask = +BIT_ULL (IB_USER_VERBS_CMD_GET_CONTEXT ) |BIT_ULL (IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL ) |BIT_ULL (IB_USER_VERBS_CMD_QUERY_DEVICE ) |BIT_ULL (IB_USER_VERBS_CMD_QUERY_PORT )
@@ -1185,6 +1186,10 @@ int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name) | BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_MW) ; + dev->uverbs_ex_cmd_mask = +BIT_ULL (IB_USER_VERBS_EX_CMD_QUERY_DEVICE ) + ; +ib_set_device_ops (dev, &rxe_dev_ops); err =ib_device_set_netdev (&rxe->ib_dev, rxe->ndev,1 );if (err)
Signature · 1 line
2.25.1
Jason Gunthorpe
maintainerRe: [PATCH for-next v7 10/19] rdma_rxe: Add support for ibv_query_device_ex
Quoted 8 lines · Bob Pearson
Add code to initialize new struct members in
ib_device_attras place holders.Signed-off-by Bob Pearson <[email protected]>--- drivers/infiniband/sw/rxe/rxe.c | 101 ++++++++++++++++++-------- drivers/infiniband/sw/rxe/rxe_verbs.c | 7 +- 2 files changed, 75 insertions(+), 33 deletions(-)
This series should eliminate this patch and notably change the others
https://patchwork.kernel.org/project/linux-rdma/list/?series=359361
Can you take a look that it works for this?
Thanks,
Jason
Re: [PATCH for-next v7 10/19] rdma_rxe: Add support for ibv_query_device_ex
Quoted 10 lines · Jason Gunthorpe
Quoted 8 lines · Bob Pearson
Add code to initialize new struct members in
ib_device_attras place holders.Signed-off-by Bob Pearson <[email protected]>--- drivers/infiniband/sw/rxe/rxe.c | 101 ++++++++++++++++++-------- drivers/infiniband/sw/rxe/rxe_verbs.c | 7 +- 2 files changed, 75 insertions(+), 33 deletions(-)
Quoted 9 lines
This series should eliminate this patch and notably change the others
https://patchwork.kernel.org/project/linux-rdma/list/?series=359361
Can you take a look that it works for this?
Thanks,
Jason
I'll take a look. Let me know if/when you take it into for-next. Should be easy to adjust for these. Currently I am working to try to improve performance. I have replaced tasklets with work queues. Low QP count performance is not changed much but it should help high QP count performance. At the moment I am trying to optimize UD since it is a lot simpler. Once that is done I want to look at core affinity.
Bob
PATCH v7 11/19 rdma_rxe: Add support for extended CQ operations
In reply to Bob PearsonAdd private members to user/kernel wc struct to carry extensions used by cq_ex.
Add timestamps on completion.
Add ignore overrun support.
Add commands to user API bitmasks.
--- drivers/infiniband/sw/rxe/rxe_comp.c | 7 ++++- drivers/infiniband/sw/rxe/rxe_resp.c | 8 +++++- drivers/infiniband/sw/rxe/rxe_verbs.c | 10 ++++--- drivers/infiniband/sw/rxe/rxe_verbs.h | 3 ++- include/uapi/rdma/rdma_user_rxe.h | 38 ++++++++++++++++++++++----- 5 files changed, 52 insertions(+), 14 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +6 -1
@@ -390,7 +390,7 @@ static void make_send_cqe(struct rxe_qp *qp, struct rxe_send_wqe *wqe, wc->byte_len = wqe->dma.length; wc->qp = &qp->ibqp; }else { -struct ib_uverbs_wc *uwc = &cqe->uibwc; +struct rxe_uverbs_wc *uwc = &cqe->ruwc; uwc->wr_id = wqe->wr.wr_id; uwc->status = wqe->status;
@@ -400,6 +400,11 @@ static void make_send_cqe(struct rxe_qp *qp, struct rxe_send_wqe *wqe, uwc->wc_flags =IB_WC_WITH_IMM ; uwc->byte_len = wqe->dma.length; uwc->qp_num = qp->ibqp.qp_num; +if (qp->scq->flags & +IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION ) { + uwc->timestamp = (u64 )ktime_get (); + uwc->realtime = (u64 )ktime_get_real (); + } } }
drivers/infiniband/sw/rxe/rxe_resp.c +7 -1
@@ -887,7 +887,7 @@ static enum resp_states do_complete(struct rxe_qp *qp, {struct rxe_cqe cqe;struct ib_wc *wc = &cqe.ibwc; -struct ib_uverbs_wc *uwc = &cqe.uibwc; +struct rxe_uverbs_wc *uwc = &cqe.ruwc;struct rxe_recv_wqe *wqe = qp->resp.wqe;struct rxe_dev *rxe =to_rdev (qp->ibqp.device);
@@ -943,6 +943,12 @@ static enum resp_states do_complete(struct rxe_qp *qp, uwc->src_qp =deth_sqp (pkt); uwc->port_num = qp->attr.port_num; + +if (qp->rcq->flags & +IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION ) { + uwc->timestamp = (u64 )ktime_get (); + uwc->realtime = (u64 )ktime_get_real (); + } }else {struct sk_buff *skb =PKT_TO_SKB (pkt);
drivers/infiniband/sw/rxe/rxe_verbs.c +6 -4
@@ -749,7 +749,8 @@ static int rxe_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,return err; } -static int rxe_create_cq (struct ib_cq *ibcq,const struct ib_cq_init_attr *attr, +static int rxe_create_cq (struct ib_cq *ibcq, +const struct ib_cq_init_attr *attr,struct ib_udata *udata) {int err;
@@ -764,13 +765,12 @@ static int rxe_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, uresp = udata->outbuf; } -if (attr->flags) -return -EINVAL ; - err =rxe_cq_chk_attr (rxe,NULL , attr->cqe, attr->comp_vector);if (err)return err; + cq->flags = attr->flags; + err =rxe_cq_from_init (rxe, cq, attr->cqe, attr->comp_vector, udata, uresp);if (err)
@@ -1188,6 +1188,8 @@ int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name) dev->uverbs_ex_cmd_mask =BIT_ULL (IB_USER_VERBS_EX_CMD_QUERY_DEVICE ) + |BIT_ULL (IB_USER_VERBS_EX_CMD_CREATE_CQ ) + |BIT_ULL (IB_USER_VERBS_EX_CMD_MODIFY_CQ ) ;ib_set_device_ops (dev, &rxe_dev_ops);
drivers/infiniband/sw/rxe/rxe_verbs.h +2 -1
@@ -53,7 +53,7 @@ struct rxe_ah {struct rxe_cqe {union {struct ib_wc ibwc; -struct ib_uverbs_wc uibwc; +struct rxe_uverbs_wc ruwc; }; };
@@ -62,6 +62,7 @@ struct rxe_cq {struct rxe_pool_entry pelem;struct rxe_queue *queue;spinlock_t cq_lock; +u32 flags;u8 notify;bool is_dying;int is_user;
include/uapi/rdma/rdma_user_rxe.h +31 -7
@@ -98,29 +98,27 @@ struct rxe_send_wr {__aligned_u64 length;union {__u32 mr_index; -__aligned_u64 reserved1; +__aligned_u64 pad1; };union {__u32 mw_index; -__aligned_u64 reserved2; +__aligned_u64 pad2; };__u32 rkey;__u32 access;__u32 flags; } umw; -/* The following are only used by the kernel -* and are not part of the uapi -*/ +/* below are only used by the kernel */ struct {__aligned_u64 addr;__aligned_u64 length;union {struct ib_mr *mr; -__aligned_u64 reserved1; +__aligned_u64 reserved1; };union {struct ib_mw *mw; -__aligned_u64 reserved2; +__aligned_u64 reserved2; };__u32 rkey;__u32 access;
@@ -184,6 +182,32 @@ struct rxe_recv_wqe {struct rxe_dma_info dma; }; +struct rxe_uverbs_wc { +/* keep these the same as ib_uverbs_wc */ +__aligned_u64 wr_id; +__u32 status; +__u32 opcode; +__u32 vendor_err; +__u32 byte_len; +union { +__be32 imm_data; +__u32 invalidate_rkey; + } ex; +__u32 qp_num; +__u32 src_qp; +__u32 wc_flags; +__u16 pkey_index; +__u16 slid; +__u8 sl; +__u8 dlid_path_bits; +__u8 port_num; +__u8 reserved; + +/* any extras go here */ +__aligned_u64 timestamp; +__aligned_u64 realtime; +}; +struct rxe_create_cq_resp {struct mminfo mi; };
Signature · 1 line
2.25.1
PATCH v7 12/19 rdma_rxe: Add support for extended QP operations
In reply to Bob PearsonAdd bits to user api command bitmask.
--- drivers/infiniband/sw/rxe/rxe_verbs.c | 2 ++ 1 file changed, 2 insertions(+)
drivers/infiniband/sw/rxe/rxe_verbs.c +2 -0
@@ -1188,6 +1188,8 @@ int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name) dev->uverbs_ex_cmd_mask =BIT_ULL (IB_USER_VERBS_EX_CMD_QUERY_DEVICE ) + |BIT_ULL (IB_USER_VERBS_EX_CMD_CREATE_QP ) + |BIT_ULL (IB_USER_VERBS_EX_CMD_MODIFY_QP ) |BIT_ULL (IB_USER_VERBS_EX_CMD_CREATE_CQ ) |BIT_ULL (IB_USER_VERBS_EX_CMD_MODIFY_CQ ) ;
Signature · 1 line
2.25.1
PATCH v7 13/19 rdma_rxe: Fix mcast group allocation bug
In reply to Bob PearsonThis patch does the following:
- Cleans up multicast group create to use an atomic sequence of lookup followed by allocate. This fixes an error that occurred when two QPs were attempting to attach to the same mcast address at the same time.
- Fixes a bug in rxe_mcast_get_grp not initializing err = 0. If the group is found in get_key the routine can return an uninitialized value to caller.
- Changes the variable elem to mce (for mcast elem). This is less likely to confuse readers because of the ambiguity with elem as pool element which is also used.
--- drivers/infiniband/sw/rxe/rxe_mcast.c | 108 ++++++++++++++------------ 1 file changed, 60 insertions(+), 48 deletions(-)
drivers/infiniband/sw/rxe/rxe_mcast.c +60 -48
@@ -7,44 +7,56 @@#include "rxe.h" #include "rxe_loc.h" -int rxe_mcast_get_grp (struct rxe_dev *rxe,union ib_gid *mgid, -struct rxe_mc_grp **grp_p) +/* caller should hold mc_grp_pool->pool_lock */ +static struct rxe_mc_grp *create_grp (struct rxe_dev *rxe, +struct rxe_pool *pool, +union ib_gid *mgid) {int err;struct rxe_mc_grp *grp; -if (rxe->attr.max_mcast_qp_attach ==0 ) { - err = -EINVAL ; - goto err1; - } - - grp =rxe_get_key (&rxe->mc_grp_pool, mgid); -if (grp) - goto done; - - grp =rxe_alloc (&rxe->mc_grp_pool); -if (!grp) { - err = -ENOMEM ; - goto err1; - } + grp =__alloc (&rxe->mc_grp_pool); +if (unlikely (!grp)) +return NULL ;INIT_LIST_HEAD (&grp->qp_list);spin_lock_init (&grp->mcg_lock); grp->rxe = rxe; - -rxe_add_key (grp, mgid); +add_key (grp, mgid); err =rxe_mcast_add (rxe, mgid); -if (err) - goto err2; +if (unlikely (err)) { +drop_key (grp); +rxe_drop_ref (grp); +return NULL ; + } +return grp; +} + +/* atomically lookup or create mc group */ +int rxe_mcast_get_grp (struct rxe_dev *rxe,union ib_gid *mgid, +struct rxe_mc_grp **grp_p) +{ +int err =0 ; +struct rxe_mc_grp *grp; +struct rxe_pool *pool = &rxe->mc_grp_pool; +unsigned long flags; + +if (unlikely (rxe->attr.max_mcast_qp_attach ==0 )) +return -EINVAL ; + +write_lock_irqsave (&pool->pool_lock, flags); + grp =__get_key (pool, mgid); +if (grp) + goto done; + + grp =create_grp (rxe, pool, mgid); +if (unlikely (!grp)) + err = -ENOMEM ; done: +write_unlock_irqrestore (&pool->pool_lock, flags); *grp_p = grp; -return 0 ; - -err2: -rxe_drop_ref (grp); -err1:return err; }
@@ -52,13 +64,13 @@ int rxe_mcast_add_grp_elem(struct rxe_dev *rxe, struct rxe_qp *qp, struct rxe_mc_grp *grp) {int err; -struct rxe_mc_elem *elem; +struct rxe_mc_elem *mce;/* check to see of the qp is already a member of the group */ spin_lock_bh (&qp->grp_lock);spin_lock_bh (&grp->mcg_lock); -list_for_each_entry (elem, &grp->qp_list, qp_list) { -if (elem->qp == qp) { +list_for_each_entry (mce, &grp->qp_list, qp_list) { +if (mce->qp == qp) { err =0 ; goto out; }
@@ -69,8 +81,8 @@ int rxe_mcast_add_grp_elem(struct rxe_dev *rxe, struct rxe_qp *qp, goto out; } - elem =rxe_alloc (&rxe->mc_elem_pool); -if (!elem) { + mce =rxe_alloc (&rxe->mc_elem_pool); +if (!mce) { err = -ENOMEM ; goto out; }
@@ -79,11 +91,11 @@ int rxe_mcast_add_grp_elem(struct rxe_dev *rxe, struct rxe_qp *qp,rxe_add_ref (grp); grp->num_qp++; - elem->qp = qp; - elem->grp = grp; + mce->qp = qp; + mce->grp = grp; -list_add (&elem->qp_list, &grp->qp_list); -list_add (&elem->grp_list, &qp->grp_list); +list_add (&mce->qp_list, &grp->qp_list); +list_add (&mce->grp_list, &qp->grp_list); err =0 ; out:
@@ -96,7 +108,7 @@ int rxe_mcast_drop_grp_elem(struct rxe_dev *rxe, struct rxe_qp *qp, union ib_gid *mgid) {struct rxe_mc_grp *grp; -struct rxe_mc_elem *elem, *tmp; +struct rxe_mc_elem *mce, *tmp; grp =rxe_get_key (&rxe->mc_grp_pool, mgid);if (!grp)
@@ -105,15 +117,15 @@ int rxe_mcast_drop_grp_elem(struct rxe_dev *rxe, struct rxe_qp *qp,spin_lock_bh (&qp->grp_lock);spin_lock_bh (&grp->mcg_lock); -list_for_each_entry_safe (elem, tmp, &grp->qp_list, qp_list) { -if (elem->qp == qp) { -list_del (&elem->qp_list); -list_del (&elem->grp_list); +list_for_each_entry_safe (mce, tmp, &grp->qp_list, qp_list) { +if (mce->qp == qp) { +list_del (&mce->qp_list); +list_del (&mce->grp_list); grp->num_qp--;spin_unlock_bh (&grp->mcg_lock);spin_unlock_bh (&qp->grp_lock); -rxe_drop_ref (elem); +rxe_drop_ref (mce);rxe_drop_ref (grp);/* ref held by QP */ rxe_drop_ref (grp);/* ref from get_key */ return 0 ;
@@ -130,7 +142,7 @@ int rxe_mcast_drop_grp_elem(struct rxe_dev *rxe, struct rxe_qp *qp,void rxe_drop_all_mcast_groups (struct rxe_qp *qp) {struct rxe_mc_grp *grp; -struct rxe_mc_elem *elem; +struct rxe_mc_elem *mce;while (1 ) {spin_lock_bh (&qp->grp_lock);
@@ -138,24 +150,24 @@ void rxe_drop_all_mcast_groups(struct rxe_qp *qp)spin_unlock_bh (&qp->grp_lock);break ; } - elem = list_first_entry(&qp->grp_list, struct rxe_mc_elem, + mce = list_first_entry(&qp->grp_list, struct rxe_mc_elem, grp_list); -list_del (&elem->grp_list); +list_del (&mce->grp_list);spin_unlock_bh (&qp->grp_lock); - grp = elem->grp; + grp = mce->grp;spin_lock_bh (&grp->mcg_lock); -list_del (&elem->qp_list); +list_del (&mce->qp_list); grp->num_qp--;spin_unlock_bh (&grp->mcg_lock);rxe_drop_ref (grp); -rxe_drop_ref (elem); +rxe_drop_ref (mce); } } -void rxe_mc_cleanup (struct rxe_pool_entry *arg) +void rxe_mc_cleanup (struct rxe_pool_entry *pelem) { -struct rxe_mc_grp *grp =container_of (arg,typeof (*grp), pelem); + struct rxe_mc_grp *grp = container_of(pelem, struct rxe_mc_grp, pelem);struct rxe_dev *rxe = grp->rxe;rxe_drop_key (grp);
Signature · 1 line
2.25.1
PATCH v7 14/19 rdma_rxe: Fix bugs in the multicast receive path
In reply to Bob PearsonThis patch does the following
- Fix a bug in rxe_rcv.
The current code calls rxe_match_dgid which checks to see if the destination ip address (dgid) matches one of the addresses in the gid table. This is ok for unicast adfdresses but not mcast addresses. Because of this all mcast packets were previously dropped.
- Fix a bug in rxe_rcv_mcast_pkt.
The current code is just wrong. It assumed that it could pass the same skb to rxe_rcv_pkt changing the qp pointer as it went when multiple QPs were attached to the same mcast address. In fact each QP needs a separate clone of the skb which it will delete later.
--- drivers/infiniband/sw/rxe/rxe_recv.c | 61 ++++++++++++++++------------ 1 file changed, 36 insertions(+), 25 deletions(-)
drivers/infiniband/sw/rxe/rxe_recv.c +36 -25
@@ -233,6 +233,8 @@ static void rxe_rcv_mcast_pkt(struct rxe_dev *rxe, struct sk_buff *skb)struct rxe_mc_elem *mce;struct rxe_qp *qp;union ib_gid dgid; +struct sk_buff *per_qp_skb; +struct rxe_pkt_info *per_qp_pkt;int err;if (skb->protocol ==htons (ETH_P_IP ))
@@ -261,42 +263,36 @@ static void rxe_rcv_mcast_pkt(struct rxe_dev *rxe, struct sk_buff *skb)if (err)continue ; -/* if *not* the last qp in the list -* increase the users of the skb then post to the next qp +/* for all but the last qp create a new clone of the +* skb and pass to the qp. +* This effectively reverts an earlier change +* which did not work. The pkt struct is contained +* in the skb so each time you changed pkt you also +* changed all the earlier pkts as well. Caused a mess. */ if (mce->qp_list.next != &mcg->qp_list) -skb_get (skb); + per_qp_skb =skb_clone (skb,GFP_ATOMIC ); +else + per_qp_skb = skb; - pkt->qp = qp; + per_qp_pkt =SKB_TO_PKT (per_qp_skb); + per_qp_pkt->qp = qp;rxe_add_ref (qp); -rxe_rcv_pkt (pkt, skb); +rxe_rcv_pkt (per_qp_pkt, per_qp_skb); }spin_unlock_bh (&mcg->mcg_lock); - -rxe_drop_ref (mcg);/* drop ref from rxe_get_key. */ +rxe_drop_ref (mcg);/* drop ref from rxe_pool_get_key. */ +return ; err1:kfree_skb (skb); } -static int rxe_match_dgid (struct rxe_dev *rxe,struct sk_buff *skb) +static int rxe_match_dgid (struct rxe_dev *rxe,struct sk_buff *skb, +union ib_gid *pdgid) { -struct rxe_pkt_info *pkt =SKB_TO_PKT (skb);const struct ib_gid_attr *gid_attr; -union ib_gid dgid; -union ib_gid *pdgid; - -if (pkt->mask &RXE_LOOPBACK_MASK ) -return 0 ; - -if (skb->protocol ==htons (ETH_P_IP )) { -ipv6_addr_set_v4mapped (ip_hdr (skb)->daddr, - (struct in6_addr *)&dgid); - pdgid = &dgid; - }else { - pdgid = (union ib_gid *)&ipv6_hdr (skb)->daddr; - } gid_attr = rdma_find_gid_by_port(&rxe->ib_dev, pdgid, IB_GID_TYPE_ROCE_UDP_ENCAP,
@@ -314,17 +310,32 @@ void rxe_rcv(struct sk_buff *skb)int err;struct rxe_pkt_info *pkt =SKB_TO_PKT (skb);struct rxe_dev *rxe = pkt->rxe; +union ib_gid dgid; +union ib_gid *pdgid;__be32 *icrcp;u32 calc_icrc, pack_icrc; +int is_mc; pkt->offset =0 ;if (unlikely (skb->len < pkt->offset +RXE_BTH_BYTES )) goto drop; -if (rxe_match_dgid (rxe, skb) <0 ) { -pr_warn_ratelimited ("failed matching dgid\n" ); - goto drop; +if (skb->protocol ==htons (ETH_P_IP )) { +ipv6_addr_set_v4mapped (ip_hdr (skb)->daddr, + (struct in6_addr *)&dgid); + pdgid = &dgid; + }else { + pdgid = (union ib_gid *)&ipv6_hdr (skb)->daddr; + } + + is_mc =rdma_is_multicast_addr ((struct in6_addr *)pdgid); + +if (!(pkt->mask &RXE_LOOPBACK_MASK ) && !is_mc) { +if (rxe_match_dgid (rxe, skb, pdgid) <0 ) { +pr_warn_ratelimited ("failed matching dgid\n" ); + goto drop; + } } pkt->opcode =bth_opcode (pkt);
Signature · 1 line
2.25.1
PATCH v7 15/19 rdma_rxe: handle ERR_PTR returns from pool
In reply to Bob Pearsonv7:
- rxe_pool.c has been extended to return errors from alloc, get_index and get_key using ERR_PTR. Add code to detect these as needed.
--- drivers/infiniband/sw/rxe/rxe_mcast.c | 28 ++++++++++++++++++--------- drivers/infiniband/sw/rxe/rxe_mr.c | 2 +- drivers/infiniband/sw/rxe/rxe_mw.c | 11 +++++++++++ drivers/infiniband/sw/rxe/rxe_pool.c | 6 +++--- drivers/infiniband/sw/rxe/rxe_recv.c | 4 +++- drivers/infiniband/sw/rxe/rxe_req.c | 4 ++-- drivers/infiniband/sw/rxe/rxe_resp.c | 8 ++++---- drivers/infiniband/sw/rxe/rxe_verbs.c | 16 +++++++-------- 8 files changed, 51 insertions(+), 28 deletions(-)
drivers/infiniband/sw/rxe/rxe_mcast.c +19 -9
@@ -16,8 +16,8 @@ static struct rxe_mc_grp *create_grp(struct rxe_dev *rxe,struct rxe_mc_grp *grp; grp =__alloc (&rxe->mc_grp_pool); -if (unlikely (!grp)) -return NULL ; +if (IS_ERR (grp)) +return grp;INIT_LIST_HEAD (&grp->qp_list);spin_lock_init (&grp->mcg_lock);
@@ -28,7 +28,7 @@ static struct rxe_mc_grp *create_grp(struct rxe_dev *rxe,if (unlikely (err)) {drop_key (grp);rxe_drop_ref (grp); -return NULL ; +return ERR_PTR (err); }return grp;
@@ -43,20 +43,30 @@ int rxe_mcast_get_grp(struct rxe_dev *rxe, union ib_gid *mgid,struct rxe_pool *pool = &rxe->mc_grp_pool;unsigned long flags; -if (unlikely (rxe->attr.max_mcast_qp_attach ==0 )) -return -EINVAL ; +if (unlikely (rxe->attr.max_mcast_qp_attach ==0 )) { + err = -EINVAL ; + goto err; + }write_lock_irqsave (&pool->pool_lock, flags); grp =__get_key (pool, mgid); +if (IS_ERR (grp)) + goto err_ptr;if (grp) goto done; grp =create_grp (rxe, pool, mgid); -if (unlikely (!grp)) - err = -ENOMEM ; +if (IS_ERR (grp)) + goto err_ptr; done:write_unlock_irqrestore (&pool->pool_lock, flags); *grp_p = grp; +return 0 ; + +err_ptr: +write_unlock_irqrestore (&pool->pool_lock, flags); + err =PTR_ERR (grp); +err:return err; }
@@ -82,8 +92,8 @@ int rxe_mcast_add_grp_elem(struct rxe_dev *rxe, struct rxe_qp *qp, } mce =rxe_alloc (&rxe->mc_elem_pool); -if (!mce) { - err = -ENOMEM ; +if (IS_ERR (mce)) { + err =PTR_ERR (mce); goto out; }
drivers/infiniband/sw/rxe/rxe_mr.c +1 -1
@@ -386,7 +386,7 @@ static struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 lkey)struct rxe_dev *rxe =to_rdev (pd->ibpd.device); mr =rxe_get_key (&rxe->mr_pool, &lkey); -if (!mr) +if (IS_ERR (mr) || !mr)return NULL ; if (unlikely((mr->ibmr.lkey != lkey) || (mr->pd != pd) ||
drivers/infiniband/sw/rxe/rxe_mw.c +11 -0
@@ -217,6 +217,8 @@ static int do_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe, rkey = mw->ibmw.rkey; new_rkey = (rkey &0xffffff00 ) | (wqe->wr.wr.umw.rkey &0x000000ff ); duplicate_mw =rxe_get_key (&rxe->mw_pool, &new_rkey); +if (IS_ERR (duplicate_mw)) +return PTR_ERR (duplicate_mw);if (duplicate_mw) {pr_err_once ("new MW key is a duplicate, try another\n" );rxe_drop_ref (duplicate_mw);
@@ -260,14 +262,23 @@ int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe)if (qp->is_user) { mw =rxe_get_index (&rxe->mw_pool, wqe->wr.wr.umw.mw_index); +if (IS_ERR (mw)) { + ret =PTR_ERR (mw); + goto err1; + }if (!mw) {pr_err_once ("mw with index = %d not found\n" , wqe->wr.wr.umw.mw_index); ret = -EINVAL ; goto err1; } + mr =rxe_get_index (&rxe->mr_pool, wqe->wr.wr.umw.mr_index); +if (IS_ERR (mr)) { + ret =PTR_ERR (mr); + goto err2; + }if (!mr && wqe->wr.wr.umw.length) {pr_err_once ("mr with index = %d not found\n" , wqe->wr.wr.umw.mr_index);
drivers/infiniband/sw/rxe/rxe_pool.c +3 -3
@@ -581,7 +581,7 @@ void *rxe_alloc(struct rxe_pool *pool) (pool->flags &RXE_POOL_ATOMIC ) ? GFP_ATOMIC : GFP_KERNEL);if (!obj) - goto out; +return ERR_PTR (-ENOMEM ); elem = (struct rxe_pool_entry *)((u8 *)obj + rxe_type_info[pool->type].elem_offset);
@@ -591,9 +591,9 @@ void *rxe_alloc(struct rxe_pool *pool)write_unlock_irqrestore (&pool->pool_lock, flags);if (err) {kfree (obj); - obj =NULL ; +return ERR_PTR (err); } -out: +return obj; }
drivers/infiniband/sw/rxe/rxe_recv.c +3 -1
@@ -186,6 +186,8 @@ static int hdr_check(struct rxe_pkt_info *pkt) index = (qpn ==1 ) ? port->qp_gsi_index : qpn; qp =rxe_get_index (&rxe->qp_pool, index); +if (IS_ERR (qp)) + goto err1;if (unlikely (!qp)) {pr_warn_ratelimited ("no qp matches qpn 0x%x\n" , qpn); goto err1;
@@ -245,7 +247,7 @@ static void rxe_rcv_mcast_pkt(struct rxe_dev *rxe, struct sk_buff *skb)/* lookup mcast group corresponding to mgid, takes a ref */ mcg =rxe_get_key (&rxe->mc_grp_pool, &dgid); -if (!mcg) +if (IS_ERR (mcg) || !mcg) goto err1;/* mcast group not registered */ spin_lock_bh (&mcg->mcg_lock);
drivers/infiniband/sw/rxe/rxe_req.c +2 -2
@@ -565,7 +565,7 @@ static int invalidate_key(struct rxe_qp *qp, u32 key)if (key &IS_MW ) { mw =rxe_get_key (&rxe->mw_pool, &key); -if (!mw) { +if (IS_ERR (mw) || !mw) {pr_err ("No mw for key %#x\n" , key);return -EINVAL ; }
@@ -573,7 +573,7 @@ static int invalidate_key(struct rxe_qp *qp, u32 key)rxe_drop_ref (mw); }else { mr =rxe_get_key (&rxe->mr_pool, &key); -if (!mr) { +if (IS_ERR (mr) || !mr) {pr_err ("No mr for key %#x\n" , key);return -EINVAL ; }
drivers/infiniband/sw/rxe/rxe_resp.c +4 -4
@@ -442,7 +442,7 @@ static enum resp_states check_rkey(struct rxe_qp *qp,*/ if (rkey &IS_MW ) { mw =rxe_get_key (&rxe->mw_pool, &rkey); -if (!mw) { +if (IS_ERR (mw) || !mw) {pr_err_once ("no MW found with rkey = 0x%08x\n" , rkey); state =RESPST_ERR_RKEY_VIOLATION ; goto err;
@@ -466,7 +466,7 @@ static enum resp_states check_rkey(struct rxe_qp *qp,rxe_drop_ref (mw); }else { mr =rxe_get_key (&rxe->mr_pool, &rkey); -if (!mr || (mr->rkey != rkey)) { +if (IS_ERR (mr) || !mr || (mr->rkey != rkey)) {pr_err_once ("no MR found with rkey = 0x%08x\n" , rkey); state =RESPST_ERR_RKEY_VIOLATION ; goto err;
@@ -794,7 +794,7 @@ static int invalidate_rkey(struct rxe_qp *qp, u32 rkey)if (rkey &IS_MW ) { mw =rxe_get_key (&rxe->mw_pool, &rkey); -if (!mw) { +if (IS_ERR (mw) || !mw) {pr_err ("No mw for rkey %#x\n" , rkey); goto err; }
@@ -802,7 +802,7 @@ static int invalidate_rkey(struct rxe_qp *qp, u32 rkey)rxe_drop_ref (mw); }else { mr =rxe_get_key (&rxe->mr_pool, &rkey); -if (!mr || mr->ibmr.rkey != rkey) { +if (IS_ERR (mr) || !mr || mr->ibmr.rkey != rkey) {pr_err ("No mr for rkey %#x\n" , rkey); goto err; }
drivers/infiniband/sw/rxe/rxe_verbs.c +8 -8
@@ -397,8 +397,8 @@ static struct ib_qp *rxe_create_qp(struct ib_pd *ibpd, goto err1; qp =rxe_alloc (&rxe->qp_pool); -if (!qp) { - err = -ENOMEM ; +if (IS_ERR (qp)) { + err =PTR_ERR (qp); goto err1; }
@@ -872,9 +872,9 @@ static struct ib_mr *rxe_get_dma_mr(struct ib_pd *ibpd, int access)rxe_add_ref (pd); mr =rxe_alloc (&rxe->mr_pool); -if (!mr) { +if (IS_ERR (mr)) {rxe_drop_ref (pd); -return ERR_PTR (-ENOMEM ); +return (void *)mr; }rxe_mr_init_dma (pd, access, mr);
@@ -905,8 +905,8 @@ static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd,rxe_add_ref (pd); mr =rxe_alloc (&rxe->mr_pool); -if (!mr) { - err = -ENOMEM ; +if (IS_ERR (mr)) { + err =PTR_ERR (mr); goto err2; }
@@ -956,8 +956,8 @@ static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,rxe_add_ref (pd); mr =rxe_alloc (&rxe->mr_pool); -if (!mr) { - err = -ENOMEM ; +if (IS_ERR (mr)) { + err =PTR_ERR (mr); goto err1; }
Signature · 1 line
2.25.1
PATCH v7 16/19 rdma_rxe: remove duplicate entries in struct rxe_mr
In reply to Bob Pearsonv7:
- Struct rxe_mr had lkey and rkey values both in itself and in the struct ib_mr. These are deleted and references replaced to the ones in ibmr.
- Removed the struct rxe_pd pointer from struct rxe_mr. This duplicates the pd pointer in struct ib_mr which was also present. Added a mr_pd() macro which extracts the pd pointer from mr.
--- drivers/infiniband/sw/rxe/rxe_mr.c | 11 +++-------- drivers/infiniband/sw/rxe/rxe_req.c | 4 ++-- drivers/infiniband/sw/rxe/rxe_resp.c | 2 +- drivers/infiniband/sw/rxe/rxe_verbs.c | 2 +- drivers/infiniband/sw/rxe/rxe_verbs.h | 8 ++++---- 5 files changed, 11 insertions(+), 16 deletions(-)
drivers/infiniband/sw/rxe/rxe_mr.c +3 -8
@@ -51,8 +51,6 @@ static void rxe_mr_init(int access, struct rxe_mr *mr)if (access &IB_ACCESS_REMOTE ) mr->ibmr.rkey = mr->ibmr.lkey; - mr->lkey = mr->ibmr.lkey; - mr->rkey = mr->ibmr.rkey; mr->state =RXE_MEM_STATE_INVALID ; mr->type =RXE_MR_TYPE_NONE ; mr->map_shift =ilog2 (RXE_BUF_PER_MAP );
@@ -101,7 +99,6 @@ void rxe_mr_init_dma(struct rxe_pd *pd, {rxe_mr_init (access, mr); - mr->pd = pd; mr->access = access; mr->state =RXE_MEM_STATE_VALID ; mr->type =RXE_MR_TYPE_DMA ;
@@ -170,7 +167,6 @@ int rxe_mr_init_user(struct rxe_pd *pd, u64 start, } } - mr->pd = pd; mr->umem = umem; mr->access = access; mr->length = length;
@@ -200,7 +196,6 @@ int rxe_mr_init_fast(struct rxe_pd *pd,if (err) goto err1; - mr->pd = pd; mr->max_buf = max_pages; mr->state =RXE_MEM_STATE_FREE ; mr->type =RXE_MR_TYPE_MR ;
@@ -320,7 +315,7 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,memcpy (dest, src, length);if (crcp) - *crcp =rxe_crc32 (to_rdev (mr->pd->ibpd.device), + *crcp =rxe_crc32 (to_rdev (mr->ibmr.device), *crcp, dest, length);return 0 ;
@@ -354,7 +349,7 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,memcpy (dest, src, bytes);if (crcp) - crc =rxe_crc32 (to_rdev (mr->pd->ibpd.device), + crc =rxe_crc32 (to_rdev (mr->ibmr.device), crc, dest, bytes); length -= bytes;
@@ -389,7 +384,7 @@ static struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 lkey)if (IS_ERR (mr) || !mr)return NULL ; -if (unlikely ((mr->ibmr.lkey != lkey) || (mr->pd != pd) || +if (unlikely ((mr_lkey (mr) != lkey) || (mr_pd (mr) != pd) || (access && !(access & mr->access)) || (mr->state !=RXE_MEM_STATE_VALID ))) {rxe_drop_ref (mr);
drivers/infiniband/sw/rxe/rxe_req.c +2 -2
@@ -638,8 +638,8 @@ int rxe_requester(void *arg) mr =to_rmr (wqe->wr.wr.reg.mr); mr->state =RXE_MEM_STATE_VALID ; mr->access = wqe->wr.wr.reg.access; - mr->lkey = wqe->wr.wr.reg.key; - mr->rkey = wqe->wr.wr.reg.key; + mr->ibmr.lkey = wqe->wr.wr.reg.key; + mr->ibmr.rkey = wqe->wr.wr.reg.key; mr->iova = wqe->wr.wr.reg.mr->iova;break ;case IB_WR_BIND_MW :
drivers/infiniband/sw/rxe/rxe_resp.c +1 -1
@@ -466,7 +466,7 @@ static enum resp_states check_rkey(struct rxe_qp *qp,rxe_drop_ref (mw); }else { mr =rxe_get_key (&rxe->mr_pool, &rkey); -if (IS_ERR (mr) || !mr || (mr->rkey != rkey)) { +if (IS_ERR (mr) || !mr || (mr_rkey (mr) != rkey)) {pr_err_once ("no MR found with rkey = 0x%08x\n" , rkey); state =RESPST_ERR_RKEY_VIOLATION ; goto err;
drivers/infiniband/sw/rxe/rxe_verbs.c +1 -1
@@ -937,7 +937,7 @@ static int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)struct rxe_mr *mr =to_rmr (ibmr); mr->state =RXE_MEM_STATE_ZOMBIE ; -rxe_drop_ref (mr->pd); +rxe_drop_ref (mr_pd (mr));rxe_drop_ref (mr);return 0 ; }
drivers/infiniband/sw/rxe/rxe_verbs.h +4 -4
@@ -293,12 +293,8 @@ struct rxe_mr {struct rxe_pool_entry pelem;struct ib_mr ibmr; -struct rxe_pd *pd;struct ib_umem *umem; -u32 lkey; -u32 rkey; -enum rxe_mem_state state;enum rxe_mr_type type;u64 va;
@@ -323,6 +319,10 @@ struct rxe_mr {struct rxe_map **map; }; +#define mr_pd (mr) to_rpd((mr)->ibmr.pd) +#define mr_lkey (mr) ((mr)->ibmr.lkey) +#define mr_rkey (mr) ((mr)->ibmr.rkey) +enum rxe_send_flags {/* flag indicaes bind call came through verbs API */ RXE_BIND_MW = (1 <<0 ),
Signature · 1 line
2.25.1
PATCH v7 17/19 rdma_rme: removed unused RXE_MR_TYPE_FMR
In reply to Bob PearsonThis is a left over from the past. A step in replacing rxe mr types with ib mr types.
--- drivers/infiniband/sw/rxe/rxe_mr.c | 1 - drivers/infiniband/sw/rxe/rxe_verbs.h | 1 - 2 files changed, 2 deletions(-)
drivers/infiniband/sw/rxe/rxe_mr.c +0 -1
@@ -28,7 +28,6 @@ static int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)return 0 ;case RXE_MR_TYPE_MR : -case RXE_MR_TYPE_FMR :if (iova < mr->iova || length > mr->length || iova > mr->iova + mr->length - length)
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -1
@@ -275,7 +275,6 @@ enum rxe_mr_type {RXE_MR_TYPE_NONE ,RXE_MR_TYPE_DMA , RXE_MR_TYPE_MR, - RXE_MR_TYPE_FMR, };#define RXE_BUF_PER_MAP (PAGE_SIZE / sizeof(struct rxe_phys_buf))
Signature · 1 line
2.25.1
PATCH v7 18/19 rdma_rxe: add rkey validation checks for MR and MW
In reply to Bob PearsonAdded checks from 9.4.1.1.3 r_key validation for remote memory invalidate to MR and MW invalidate routines.
--- drivers/infiniband/sw/rxe/rxe_mr.c | 49 ++++++++++++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_mw.c | 10 +++++- 2 files changed, 58 insertions(+), 1 deletion(-)
drivers/infiniband/sw/rxe/rxe_mr.c +49 -0
@@ -522,8 +522,57 @@ int advance_dma_data(struct rxe_dma_info *dma, unsigned int length)return 0 ; } +static int check_invalidate_mr (struct rxe_qp *qp,struct rxe_mr *mr) +{ +int type = mr->ibmr.type; +int qpt =qp_type (qp); + +if (unlikely (type >IB_MR_TYPE_INTEGRITY )) { +pr_err ("invalidate: MR of unknown type = %d\n" , + mr->ibmr.type); +return -EINVAL ; + } + +/* o10-37.2.13 */ +if (unlikely (type ==IB_MR_TYPE_MEM_REG || + type ==IB_MR_TYPE_USER )) { +pr_err ("invalidate: MR created by reg_mr or user_reg_mr\n" ); +return -EINVAL ; + } + +/* o10-37.2.17 */ +if (unlikely (atomic_read (&mr->num_mw))) { +pr_err ("invalidate: MR with bound MWs\n" ); +return -EINVAL ; + } + +if (unlikely (!((qpt ==IB_QPT_RC ) || (qpt ==IB_QPT_UC ) || + (qpt ==IB_QPT_XRC_INI ) || (qpt ==IB_QPT_XRC_TGT )))) { +pr_err ("invalidate: MR with invalid QP type\n" ); +return -EINVAL ; + } + +if (unlikely (qp->ibqp.pd != mr->ibmr.pd)) { +pr_err ("invalidate: MR and QP have different PDs\n" ); +return -EINVAL ; + } + +if (unlikely (mr->state ==RXE_MEM_STATE_INVALID )) { +pr_err ("invalidate: MR in invalid state\n" ); +return -EINVAL ; + } + +return 0 ; +} +int rxe_invalidate_mr (struct rxe_qp *qp,struct rxe_mr *mr) { +int ret; + + ret =check_invalidate_mr (qp, mr); +if (ret) +return ret; +/* TODO there are API rules being ignored here * cleanup later. Current project is not trying * to fix MR
drivers/infiniband/sw/rxe/rxe_mw.c +9 -1
@@ -316,9 +316,17 @@ int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe)static int check_invalidate_mw (struct rxe_qp *qp,struct rxe_mw *mw) { +int qpt =qp_type (qp); +/* o10-37.2.26 */ if (unlikely (mw->ibmw.type ==IB_MW_TYPE_1 )) { -pr_err_once ("attempt to invalidate a type 1 MW\n" ); +pr_err_once ("invalidate a type 1 MW\n" ); +return -EINVAL ; + } + +if (unlikely (!((qpt ==IB_QPT_RC ) || (qpt ==IB_QPT_UC ) || + (qpt ==IB_QPT_XRC_INI ) || (qpt ==IB_QPT_XRC_TGT )))) { +pr_err ("invalidate Mw with invalid QP type\n" );return -EINVAL ; }
Signature · 1 line
2.25.1
PATCH v7 19/19 rdma_rxe: moved rxe_xmit_packet to rxe_net.c
In reply to Bob PearsonNo good reason to stick such a large subroutine in a header file as an inline function.
--- drivers/infiniband/sw/rxe/rxe_loc.h | 47 ++--------------------------- drivers/infiniband/sw/rxe/rxe_net.c | 47 +++++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 47 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +2 -45
@@ -124,8 +124,6 @@ int rxe_mw_check_access(struct rxe_qp *qp, struct rxe_mw *mw,void rxe_mw_cleanup (struct rxe_pool_entry *arg);/* rxe_net.c */ -void rxe_loopback (struct sk_buff *skb); -int rxe_send (struct rxe_pkt_info *pkt,struct sk_buff *skb);struct sk_buff *rxe_init_packet (struct rxe_dev *rxe,struct rxe_av *av,int paylen,struct rxe_pkt_info *pkt);int rxe_prepare (struct rxe_pkt_info *pkt,struct sk_buff *skb,u32 *crc);
@@ -133,6 +131,8 @@ const char *rxe_parent_name(struct rxe_dev *rxe, unsigned int port_num);struct device *rxe_dma_device (struct rxe_dev *rxe);int rxe_mcast_add (struct rxe_dev *rxe,union ib_gid *mgid);int rxe_mcast_delete (struct rxe_dev *rxe,union ib_gid *mgid); +int rxe_xmit_packet (struct rxe_qp *qp,struct rxe_pkt_info *pkt, +struct sk_buff *skb);/* rxe_qp.c */ int rxe_qp_chk_init (struct rxe_dev *rxe,struct ib_qp_init_attr *init);
@@ -230,47 +230,4 @@ static inline unsigned int wr_opcode_mask(int opcode, struct rxe_qp *qp) return rxe_wr_opcode_info[opcode].mask[qp->ibqp.qp_type]; } -static inline int rxe_xmit_packet (struct rxe_qp *qp,struct rxe_pkt_info *pkt, -struct sk_buff *skb) -{ -int err; -int is_request = pkt->mask &RXE_REQ_MASK ; -struct rxe_dev *rxe =to_rdev (qp->ibqp.device); - -if ((is_request && (qp->req.state !=QP_STATE_READY )) || - (!is_request && (qp->resp.state !=QP_STATE_READY ))) { -pr_info ("Packet dropped. QP is not in ready state\n" ); - goto drop; - } - -if (pkt->mask &RXE_LOOPBACK_MASK ) { -memcpy (SKB_TO_PKT (skb), pkt,sizeof (*pkt)); -rxe_loopback (skb); - err =0 ; - }else { - err =rxe_send (pkt, skb); - } - -if (err) { - rxe->xmit_errors++; -rxe_counter_inc (rxe,RXE_CNT_SEND_ERR ); -return err; - } - -if ((qp_type (qp) !=IB_QPT_RC ) && - (pkt->mask &RXE_END_MASK )) { - pkt->wqe->state = wqe_state_done; -rxe_run_task (&qp->comp.task,1 ); - } - -rxe_counter_inc (rxe,RXE_CNT_SENT_PKTS ); - goto done; - -drop: -kfree_skb (skb); - err =0 ; -done: -return err; -} -#endif /* RXE_LOC_H */
drivers/infiniband/sw/rxe/rxe_net.c +45 -2
63 lines not on this page · raw message
Signature · 1 line
2.25.1