Correct race conditions in rdma_rxe
Subsystems (2)
v2 cover Correct race conditions in rdma_rxe
There are several potential race conditions discovered in the rdma_rxe driver. These patches correct them. They mostly relate to races between normal operations and destroying objects.
This patch series applies cleanly to current for-next. commit 3b87e0824272414cec79763afef6720c7c908c44 (for-next)
---
v2
Rebased to current for-next.
Added 4 additional patches
Bob Pearson (10):
RDMA/rxe: Make rxe_alloc() take pool lock RDMA/rxe: Copy setup parameters into rxe_pool RDMA/rxe: Save object pointer in pool element RDMA/rxe: Combine rxe_add_index with rxe_alloc RDMA/rxe: Combine rxe_add_key with rxe_alloc RDMA/rxe: Fix potential race condition in rxe_pool RDMA/rxe: Separate out last rxe_drop_ref RDMA/rxe: Rewrite rxe_mcast.c
RDMA/rxe: Fix ref error in rxe_av.c RDMA/rxe: Replace mr by rkey in responder resources
drivers/infiniband/sw/rxe/rxe.c | 8 - drivers/infiniband/sw/rxe/rxe_av.c | 24 +- drivers/infiniband/sw/rxe/rxe_cq.c | 9 +- drivers/infiniband/sw/rxe/rxe_loc.h | 19 +- drivers/infiniband/sw/rxe/rxe_mcast.c | 210 ++++++++++----- drivers/infiniband/sw/rxe/rxe_mr.c | 11 +- drivers/infiniband/sw/rxe/rxe_mw.c | 28 +- drivers/infiniband/sw/rxe/rxe_net.c | 39 +-- drivers/infiniband/sw/rxe/rxe_pool.c | 374 ++++++++++++++++---------- drivers/infiniband/sw/rxe/rxe_pool.h | 86 +++--- drivers/infiniband/sw/rxe/rxe_qp.c | 11 +- drivers/infiniband/sw/rxe/rxe_req.c | 55 ++-- drivers/infiniband/sw/rxe/rxe_resp.c | 125 ++++++--- drivers/infiniband/sw/rxe/rxe_srq.c | 8 + drivers/infiniband/sw/rxe/rxe_verbs.c | 65 ++--- drivers/infiniband/sw/rxe/rxe_verbs.h | 4 +- 16 files changed, 611 insertions(+), 465 deletions(-)
Signature · 1 line
2.30.2
PATCH v2 1/10 RDMA/rxe: Make rxe_alloc() take pool lock
In rxe_pool.c there are two separate pool APIs for creating a new object rxe_alloc() and rxe_alloc_locked(). Currently they are identical. Make rxe_alloc() take the pool lock which is in line with the other APIs in the library and was the original intent.
--- drivers/infiniband/sw/rxe/rxe_pool.c | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-)
drivers/infiniband/sw/rxe/rxe_pool.c +4 -17
@@ -354,27 +354,14 @@ void *rxe_alloc_locked(struct rxe_pool *pool)void *rxe_alloc (struct rxe_pool *pool) { -struct rxe_type_info *info = &rxe_type_info[pool->type]; -struct rxe_pool_entry *elem; +unsigned long flags;u8 *obj; -if (atomic_inc_return (&pool->num_elem) > pool->max_elem) - goto out_cnt; - - obj =kzalloc (info->size,GFP_KERNEL ); -if (!obj) - goto out_cnt; - - elem = (struct rxe_pool_entry *)(obj + info->elem_offset); - - elem->pool = pool; -kref_init (&elem->ref_cnt); +write_lock_irqsave (&pool->pool_lock, flags); + obj =rxe_alloc_locked (pool); +write_unlock_irqrestore (&pool->pool_lock, flags);return obj; - -out_cnt: -atomic_dec (&pool->num_elem); -return NULL ; }int __rxe_add_to_pool (struct rxe_pool *pool,struct rxe_pool_entry *elem)
Signature · 1 line
2.30.2
PATCH v2 2/10 RDMA/rxe: Copy setup parameters into rxe_pool
In reply to Bob PearsonIn rxe_pool.c copy remaining pool setup parameters from rxe_pool_info into rxe_pool. This saves looking up rxe_pool_info in the performance path.
--- drivers/infiniband/sw/rxe/rxe_pool.c | 49 ++++++++++++---------------- drivers/infiniband/sw/rxe/rxe_pool.h | 6 ++-- 2 files changed, 23 insertions(+), 32 deletions(-)
drivers/infiniband/sw/rxe/rxe_pool.c +20 -29
@@ -7,9 +7,8 @@#include "rxe.h" #include "rxe_loc.h" -/* info about object pools -*/ -struct rxe_type_info rxe_type_info[RXE_NUM_TYPES ] = { +/* info about object pools */ +static const struct rxe_type_info rxe_type_info[RXE_NUM_TYPES ] = { [RXE_TYPE_UC ] = { .name ="rxe-uc" , .size =sizeof (struct rxe_ucontext ),
@@ -88,11 +87,6 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = { }, }; -static inline const char *pool_name (struct rxe_pool *pool) -{ -return rxe_type_info[pool->type].name; -} -static int rxe_pool_init_index (struct rxe_pool *pool,u32 max,u32 min) {int err =0 ;
@@ -127,35 +121,36 @@ int rxe_pool_init(enum rxe_elem_type type, unsigned int max_elem) { + const struct rxe_type_info *info = &rxe_type_info[type];int err =0 ; -size_t size = rxe_type_info[type].size;memset (pool,0 ,sizeof (*pool)); pool->rxe = rxe; + pool->name = info->name; pool->type = type; pool->max_elem = max_elem; - pool->elem_size =ALIGN (size,RXE_POOL_ALIGN ); - pool->flags = rxe_type_info[type].flags; + pool->elem_size =ALIGN (info->size,RXE_POOL_ALIGN ); + pool->elem_offset = info->elem_offset; + pool->flags = info->flags; pool->index.tree =RB_ROOT ; pool->key.tree =RB_ROOT ; - pool->cleanup = rxe_type_info[type].cleanup; + pool->cleanup = info->cleanup;atomic_set (&pool->num_elem,0 );rwlock_init (&pool->pool_lock); -if (rxe_type_info[type].flags &RXE_POOL_INDEX ) { - err =rxe_pool_init_index (pool, - rxe_type_info[type].max_index, - rxe_type_info[type].min_index); +if (info->flags &RXE_POOL_INDEX ) { + err =rxe_pool_init_index (pool, info->max_index, + info->min_index);if (err) goto out; } -if (rxe_type_info[type].flags &RXE_POOL_KEY ) { - pool->key.key_offset = rxe_type_info[type].key_offset; - pool->key.key_size = rxe_type_info[type].key_size; +if (info->flags &RXE_POOL_KEY ) { + pool->key.key_offset = info->key_offset; + pool->key.key_size = info->key_size; } out:
@@ -166,7 +161,7 @@ void rxe_pool_cleanup(struct rxe_pool *pool) {if (atomic_read (&pool->num_elem) >0 )pr_warn ("%s pool destroyed with unfree'd elem\n" , -pool_name (pool)); + pool->name);kfree (pool->index.table); }
@@ -329,18 +324,17 @@ void __rxe_drop_index(struct rxe_pool_entry *elem)void *rxe_alloc_locked (struct rxe_pool *pool) { -struct rxe_type_info *info = &rxe_type_info[pool->type];struct rxe_pool_entry *elem;u8 *obj;if (atomic_inc_return (&pool->num_elem) > pool->max_elem) goto out_cnt; - obj =kzalloc (info->size,GFP_ATOMIC ); + obj =kzalloc (pool->elem_size,GFP_ATOMIC );if (!obj) goto out_cnt; - elem = (struct rxe_pool_entry *)(obj + info->elem_offset); + elem = (struct rxe_pool_entry *)(obj + pool->elem_offset); elem->pool = pool;kref_init (&elem->ref_cnt);
@@ -384,14 +378,13 @@ void rxe_elem_release(struct kref *kref)struct rxe_pool_entry *elem = container_of(kref, struct rxe_pool_entry, ref_cnt);struct rxe_pool *pool = elem->pool; -struct rxe_type_info *info = &rxe_type_info[pool->type];u8 *obj;if (pool->cleanup) pool->cleanup (elem);if (!(pool->flags &RXE_POOL_NO_ALLOC )) { - obj = (u8 *)elem - info->elem_offset; + obj = (u8 *)elem - pool->elem_offset;kfree (obj); }
@@ -400,7 +393,6 @@ void rxe_elem_release(struct kref *kref)void *rxe_pool_get_index_locked (struct rxe_pool *pool,u32 index) { -struct rxe_type_info *info = &rxe_type_info[pool->type];struct rb_node *node;struct rxe_pool_entry *elem;u8 *obj;
@@ -420,7 +412,7 @@ void *rxe_pool_get_index_locked(struct rxe_pool *pool, u32 index)if (node) {kref_get (&elem->ref_cnt); - obj = (u8 *)elem - info->elem_offset; + obj = (u8 *)elem - pool->elem_offset; }else { obj =NULL ; }
@@ -442,7 +434,6 @@ void *rxe_pool_get_index(struct rxe_pool *pool, u32 index)void *rxe_pool_get_key_locked (struct rxe_pool *pool,void *key) { -struct rxe_type_info *info = &rxe_type_info[pool->type];struct rb_node *node;struct rxe_pool_entry *elem;u8 *obj;
@@ -466,7 +457,7 @@ void *rxe_pool_get_key_locked(struct rxe_pool *pool, void *key)if (node) {kref_get (&elem->ref_cnt); - obj = (u8 *)elem - info->elem_offset; + obj = (u8 *)elem - pool->elem_offset; }else { obj =NULL ; }
drivers/infiniband/sw/rxe/rxe_pool.h +3 -3
@@ -44,8 +44,6 @@ struct rxe_type_info {size_t key_size; }; -extern struct rxe_type_info rxe_type_info[]; -struct rxe_pool_entry {struct rxe_pool *pool;struct kref ref_cnt;
@@ -61,14 +59,16 @@ struct rxe_pool_entry {struct rxe_pool {struct rxe_dev *rxe; +const char *name;rwlock_t pool_lock;/* protects pool add/del/search */ -size_t elem_size;void (*cleanup)(struct rxe_pool_entry *obj);enum rxe_pool_flags flags;enum rxe_elem_type type;unsigned int max_elem;atomic_t num_elem; +size_t elem_size; +size_t elem_offset;/* only used if indexed */ struct {
Signature · 1 line
2.30.2
PATCH v2 3/10 RDMA/rxe: Save object pointer in pool element
In reply to Bob PearsonIn rxe_pool.c currently there are many cases where it is necessary to compute the offset from a pool element struct to the object containing the pool element in a type independent way. By saving a pointer to the object when they are created extra work can be saved.
--- drivers/infiniband/sw/rxe/rxe_pool.c | 25 ++++++++++++++----------- drivers/infiniband/sw/rxe/rxe_pool.h | 1 + 2 files changed, 15 insertions(+), 11 deletions(-)
drivers/infiniband/sw/rxe/rxe_pool.c +14 -11
@@ -220,7 +220,8 @@ static int rxe_insert_key(struct rxe_pool *pool, struct rxe_pool_entry *new) elem = rb_entry(parent, struct rxe_pool_entry, key_node); cmp =memcmp ((u8 *)elem + pool->key.key_offset, - (u8 *)new + pool->key.key_offset, pool->key.key_size); + (u8 *)new + pool->key.key_offset, + pool->key.key_size);if (cmp ==0 ) {pr_warn ("key already exists!\n" );
@@ -325,7 +326,7 @@ void __rxe_drop_index(struct rxe_pool_entry *elem)void *rxe_alloc_locked (struct rxe_pool *pool) {struct rxe_pool_entry *elem; -u8 *obj; +void *obj;if (atomic_inc_return (&pool->num_elem) > pool->max_elem) goto out_cnt;
@@ -337,6 +338,7 @@ void *rxe_alloc_locked(struct rxe_pool *pool) elem = (struct rxe_pool_entry *)(obj + pool->elem_offset); elem->pool = pool; + elem->obj = obj;kref_init (&elem->ref_cnt);return obj;
@@ -349,7 +351,7 @@ void *rxe_alloc_locked(struct rxe_pool *pool)void *rxe_alloc (struct rxe_pool *pool) {unsigned long flags; -u8 *obj; +void *obj;write_lock_irqsave (&pool->pool_lock, flags); obj =rxe_alloc_locked (pool);
@@ -364,6 +366,7 @@ int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_entry *elem) goto out_cnt; elem->pool = pool; + elem->obj = (u8 *)elem - pool->elem_offset;kref_init (&elem->ref_cnt);return 0 ;
@@ -378,13 +381,13 @@ void rxe_elem_release(struct kref *kref)struct rxe_pool_entry *elem = container_of(kref, struct rxe_pool_entry, ref_cnt);struct rxe_pool *pool = elem->pool; -u8 *obj; +void *obj;if (pool->cleanup) pool->cleanup (elem);if (!(pool->flags &RXE_POOL_NO_ALLOC )) { - obj = (u8 *)elem - pool->elem_offset; + obj = elem->obj;kfree (obj); }
@@ -395,7 +398,7 @@ void *rxe_pool_get_index_locked(struct rxe_pool *pool, u32 index) {struct rb_node *node;struct rxe_pool_entry *elem; -u8 *obj; +void *obj; node = pool->index.tree.rb_node;
@@ -412,7 +415,7 @@ void *rxe_pool_get_index_locked(struct rxe_pool *pool, u32 index)if (node) {kref_get (&elem->ref_cnt); - obj = (u8 *)elem - pool->elem_offset; + obj = elem->obj; }else { obj =NULL ; }
@@ -422,7 +425,7 @@ void *rxe_pool_get_index_locked(struct rxe_pool *pool, u32 index)void *rxe_pool_get_index (struct rxe_pool *pool,u32 index) { -u8 *obj; +void *obj;unsigned long flags;read_lock_irqsave (&pool->pool_lock, flags);
@@ -436,7 +439,7 @@ void *rxe_pool_get_key_locked(struct rxe_pool *pool, void *key) {struct rb_node *node;struct rxe_pool_entry *elem; -u8 *obj; +void *obj;int cmp; node = pool->key.tree.rb_node;
@@ -457,7 +460,7 @@ void *rxe_pool_get_key_locked(struct rxe_pool *pool, void *key)if (node) {kref_get (&elem->ref_cnt); - obj = (u8 *)elem - pool->elem_offset; + obj = elem->obj; }else { obj =NULL ; }
@@ -467,7 +470,7 @@ void *rxe_pool_get_key_locked(struct rxe_pool *pool, void *key)void *rxe_pool_get_key (struct rxe_pool *pool,void *key) { -u8 *obj; +void *obj;unsigned long flags;read_lock_irqsave (&pool->pool_lock, flags);
drivers/infiniband/sw/rxe/rxe_pool.h +1 -0
@@ -46,6 +46,7 @@ struct rxe_type_info {struct rxe_pool_entry {struct rxe_pool *pool; +void *obj;struct kref ref_cnt;struct list_head list;
Signature · 1 line
2.30.2
PATCH v2 4/10 RDMA/rxe: Combine rxe_add_index with rxe_alloc
In reply to Bob PearsonCurrently rxe objects which have an index require adding and dropping the indices in a separate API call from allocating and freeing the object. These are always performed together so this patch combines them in a single operation.
By taking a single pool lock around allocating the object and adding the index metadata or dropping the index metadata and releasing the object the possibility of a race condition where the metadata is not consistent with the state of the object is removed.
--- drivers/infiniband/sw/rxe/rxe_mr.c | 1 - drivers/infiniband/sw/rxe/rxe_mw.c | 5 +-- drivers/infiniband/sw/rxe/rxe_pool.c | 59 +++++++++++++++------------ drivers/infiniband/sw/rxe/rxe_pool.h | 22 ---------- drivers/infiniband/sw/rxe/rxe_verbs.c | 13 ------ 5 files changed, 33 insertions(+), 67 deletions(-)
drivers/infiniband/sw/rxe/rxe_mr.c +0 -1
@@ -693,7 +693,6 @@ int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) mr->state =RXE_MR_STATE_INVALID ;rxe_drop_ref (mr_pd (mr)); -rxe_drop_index (mr);rxe_drop_ref (mr);return 0 ;
drivers/infiniband/sw/rxe/rxe_mw.c +1 -4
@@ -20,7 +20,6 @@ int rxe_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata)return ret; } -rxe_add_index (mw); mw->rkey = ibmw->rkey = (mw->pelem.index <<8 ) |rxe_get_next_key (-1 ); mw->state = (mw->ibmw.type ==IB_MW_TYPE_2 ) ?RXE_MW_STATE_FREE :RXE_MW_STATE_VALID ;
@@ -335,7 +334,5 @@ struct rxe_mw *rxe_lookup_mw(struct rxe_qp *qp, int access, u32 rkey)void rxe_mw_cleanup (struct rxe_pool_entry *elem) { -struct rxe_mw *mw =container_of (elem,typeof (*mw), pelem); - -rxe_drop_index (mw); +/* nothing to do currently */ }
drivers/infiniband/sw/rxe/rxe_pool.c +32 -27
126 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_pool.h +0 -22
@@ -109,28 +109,6 @@ int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_entry *elem);#define rxe_add_to_pool (pool, obj) __rxe_add_to_pool(pool, &(obj)->pelem) -/* assign an index to an indexed object and insert object into -* pool's rb tree holding and not holding the pool_lock -*/ -int __rxe_add_index_locked (struct rxe_pool_entry *elem); - -#define rxe_add_index_locked (obj) __rxe_add_index_locked(&(obj)->pelem) - -int __rxe_add_index (struct rxe_pool_entry *elem); - -#define rxe_add_index (obj) __rxe_add_index(&(obj)->pelem) - -/* drop an index and remove object from rb tree -* holding and not holding the pool_lock -*/ -void __rxe_drop_index_locked (struct rxe_pool_entry *elem); - -#define rxe_drop_index_locked (obj) __rxe_drop_index_locked(&(obj)->pelem) - -void __rxe_drop_index (struct rxe_pool_entry *elem); - -#define rxe_drop_index (obj) __rxe_drop_index(&(obj)->pelem) -/* assign a key to a keyed object and insert object into * pool's rb tree holding and not holding pool_lock */
drivers/infiniband/sw/rxe/rxe_verbs.c +0 -13
@@ -181,7 +181,6 @@ static int rxe_create_ah(struct ib_ah *ibah,return err;/* create index > 0 */ -rxe_add_index (ah); ah->ah_num = ah->pelem.index;if (uresp) {
@@ -189,7 +188,6 @@ static int rxe_create_ah(struct ib_ah *ibah, err =copy_to_user (&uresp->ah_num, &ah->ah_num,sizeof (uresp->ah_num));if (err) { -rxe_drop_index (ah);rxe_drop_ref (ah);return -EFAULT ; }
@@ -230,7 +228,6 @@ static int rxe_destroy_ah(struct ib_ah *ibah, u32 flags) {struct rxe_ah *ah =to_rah (ibah); -rxe_drop_index (ah);rxe_drop_ref (ah);return 0 ; }
@@ -438,7 +435,6 @@ static int rxe_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init,if (err)return err; -rxe_add_index (qp); err =rxe_qp_from_init (rxe, qp, pd, init, uresp, ibqp->pd, udata);if (err) goto qp_init;
@@ -446,7 +442,6 @@ static int rxe_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init,return 0 ; qp_init: -rxe_drop_index (qp);rxe_drop_ref (qp);return err; }
@@ -491,7 +486,6 @@ static int rxe_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)struct rxe_qp *qp =to_rqp (ibqp);rxe_qp_destroy (qp); -rxe_drop_index (qp);rxe_drop_ref (qp);return 0 ; }
@@ -898,7 +892,6 @@ static struct ib_mr *rxe_get_dma_mr(struct ib_pd *ibpd, int access)if (!mr)return ERR_PTR (-ENOMEM ); -rxe_add_index (mr);rxe_add_ref (pd);rxe_mr_init_dma (pd, access, mr);
@@ -922,8 +915,6 @@ static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd, goto err2; } -rxe_add_index (mr); -rxe_add_ref (pd); err =rxe_mr_init_user (pd, start, length, iova, access, mr);
@@ -934,7 +925,6 @@ static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd, err3:rxe_drop_ref (pd); -rxe_drop_index (mr);rxe_drop_ref (mr); err2:return ERR_PTR (err);
@@ -957,8 +947,6 @@ static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, goto err1; } -rxe_add_index (mr); -rxe_add_ref (pd); err =rxe_mr_init_fast (pd, max_num_sg, mr);
@@ -969,7 +957,6 @@ static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, err2:rxe_drop_ref (pd); -rxe_drop_index (mr);rxe_drop_ref (mr); err1:return ERR_PTR (err);
Signature · 1 line
2.30.2
PATCH v2 5/10 RDMA/rxe: Combine rxe_add_key with rxe_alloc
In reply to Bob PearsonCurrently adding and dropping a key from a rxe object requires separate API calls from allocating and freeing the object but these are always performed together. This patch combines these into single APIs. This requires adding new rxe_allocate_with_key(_locked) APIs.
By combining allocating an object and adding key metadata inside a single locked sequence and dropping the key metadata and releasing the object the possibility of a race condition where the object state and key metadata state are inconsistent is removed.
--- drivers/infiniband/sw/rxe/rxe_mcast.c | 5 +- drivers/infiniband/sw/rxe/rxe_pool.c | 81 +++++++++++++-------------- drivers/infiniband/sw/rxe/rxe_pool.h | 24 ++------ 3 files changed, 45 insertions(+), 65 deletions(-)
drivers/infiniband/sw/rxe/rxe_mcast.c +1 -4
@@ -15,18 +15,16 @@ static struct rxe_mc_grp *create_grp(struct rxe_dev *rxe,int err;struct rxe_mc_grp *grp; - grp =rxe_alloc_locked (&rxe->mc_grp_pool); + grp =rxe_alloc_with_key_locked (&rxe->mc_grp_pool, mgid);if (!grp)return ERR_PTR (-ENOMEM );INIT_LIST_HEAD (&grp->qp_list);spin_lock_init (&grp->mcg_lock); grp->rxe = rxe; -rxe_add_key_locked (grp, mgid); err =rxe_mcast_add (rxe, mgid);if (unlikely (err)) { -rxe_drop_key_locked (grp);rxe_drop_ref (grp);return ERR_PTR (err); }
@@ -174,6 +172,5 @@ void rxe_mc_cleanup(struct rxe_pool_entry *arg)struct rxe_mc_grp *grp =container_of (arg,typeof (*grp), pelem);struct rxe_dev *rxe = grp->rxe; -rxe_drop_key (grp);rxe_mcast_delete (rxe, &grp->mgid); }
drivers/infiniband/sw/rxe/rxe_pool.c +40 -41
105 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_pool.h +4 -20
@@ -104,31 +104,15 @@ void *rxe_alloc_locked(struct rxe_pool *pool);void *rxe_alloc (struct rxe_pool *pool); +void *rxe_alloc_with_key_locked (struct rxe_pool *pool,void *key); + +void *rxe_alloc_with_key (struct rxe_pool *pool,void *key); +/* connect already allocated object to pool */ int __rxe_add_to_pool (struct rxe_pool *pool,struct rxe_pool_entry *elem);#define rxe_add_to_pool (pool, obj) __rxe_add_to_pool(pool, &(obj)->pelem) -/* assign a key to a keyed object and insert object into -* pool's rb tree holding and not holding pool_lock -*/ -int __rxe_add_key_locked (struct rxe_pool_entry *elem,void *key); - -#define rxe_add_key_locked (obj, key) __rxe_add_key_locked(&(obj)->pelem, key) - -int __rxe_add_key (struct rxe_pool_entry *elem,void *key); - -#define rxe_add_key (obj, key) __rxe_add_key(&(obj)->pelem, key) - -/* remove elem from rb tree holding and not holding the pool_lock */ -void __rxe_drop_key_locked (struct rxe_pool_entry *elem); - -#define rxe_drop_key_locked (obj) __rxe_drop_key_locked(&(obj)->pelem) - -void __rxe_drop_key (struct rxe_pool_entry *elem); - -#define rxe_drop_key (obj) __rxe_drop_key(&(obj)->pelem) -/* lookup an indexed object from index holding and not holding the pool_lock. * takes a reference on object */
Signature · 1 line
2.30.2
PATCH v2 6/10 RDMA/rxe: Fix potential race condition in rxe_pool
In reply to Bob PearsonCurrently there is a possible race condition related to rxe indexed or keyed objects where one thread is the last one holding a reference to an object and drops that reference triggering a call to rxe_elem_release() while at the same time another thread looks up the object from its index or key by calling rxe_pool_get_index(/_key). This can happen if an unexpected packet arrives as a result of a retry attempt and looks up its rkey or a multicast packet arrives just as the verbs consumer drops the mcast group.
Add locking to prevent looking up an object from its index or key while another thread is trying to destroy the object.
--- drivers/infiniband/sw/rxe/rxe_pool.c | 51 +++++++++++++++++----------- drivers/infiniband/sw/rxe/rxe_pool.h | 15 ++++++-- 2 files changed, 45 insertions(+), 21 deletions(-)
drivers/infiniband/sw/rxe/rxe_pool.c +32 -19
@@ -267,7 +267,7 @@ static void rxe_drop_index(struct rxe_pool_entry *elem)rb_erase (&elem->index_node, &pool->index.tree); } -void *rxe_alloc_locked (struct rxe_pool *pool) +static void *__rxe_alloc_locked (struct rxe_pool *pool) {struct rxe_pool_entry *elem;void *obj;
@@ -280,11 +280,10 @@ void *rxe_alloc_locked(struct rxe_pool *pool)if (!obj) goto out_cnt; - elem = (struct rxe_pool_entry *)(obj + pool->elem_offset); + elem = (struct rxe_pool_entry *)((u8 *)obj + pool->elem_offset); elem->pool = pool; elem->obj = obj; -kref_init (&elem->ref_cnt);if (pool->flags &RXE_POOL_INDEX ) { err =rxe_add_index (elem);
@@ -301,17 +300,32 @@ void *rxe_alloc_locked(struct rxe_pool *pool)return NULL ; } +void *rxe_alloc_locked (struct rxe_pool *pool) +{ +struct rxe_pool_entry *elem; +void *obj; + + obj =__rxe_alloc_locked (pool); +if (!obj) +return NULL ; + + elem = (struct rxe_pool_entry *)(obj + pool->elem_offset); +kref_init (&elem->ref_cnt); + +return obj; +} +void *rxe_alloc_with_key_locked (struct rxe_pool *pool,void *key) {struct rxe_pool_entry *elem; -u8 *obj; +void *obj;int err; - obj =rxe_alloc_locked (pool); + obj =__rxe_alloc_locked (pool);if (!obj)return NULL ; - elem = (struct rxe_pool_entry *)(obj + pool->elem_offset); + elem = (struct rxe_pool_entry *)((u8 *)obj + pool->elem_offset);memcpy ((u8 *)elem + pool->key.key_offset, key, pool->key.key_size); err =rxe_insert_key (pool, elem);if (err) {
@@ -319,6 +333,8 @@ void *rxe_alloc_with_key_locked(struct rxe_pool *pool, void *key) goto out_cnt; } +kref_init (&elem->ref_cnt); +return obj; out_cnt:
@@ -352,14 +368,15 @@ void *rxe_alloc_with_key(struct rxe_pool *pool, void *key)int __rxe_add_to_pool (struct rxe_pool *pool,struct rxe_pool_entry *elem) { +unsigned long flags;int err; +write_lock_irqsave (&pool->pool_lock, flags);if (atomic_inc_return (&pool->num_elem) > pool->max_elem) goto out_cnt; elem->pool = pool; elem->obj = (u8 *)elem - pool->elem_offset; -kref_init (&elem->ref_cnt);if (pool->flags &RXE_POOL_INDEX ) { err =rxe_add_index (elem);
@@ -367,10 +384,14 @@ int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_entry *elem) goto out_cnt; } +kref_init (&elem->ref_cnt); +write_unlock_irqrestore (&pool->pool_lock, flags); +return 0 ; out_cnt:atomic_dec (&pool->num_elem); +write_unlock_irqrestore (&pool->pool_lock, flags);return -EINVAL ; }
@@ -402,7 +423,7 @@ void *rxe_pool_get_index_locked(struct rxe_pool *pool, u32 index) {struct rb_node *node;struct rxe_pool_entry *elem; -void *obj; +void *obj =NULL ; node = pool->index.tree.rb_node;
@@ -417,12 +438,8 @@ void *rxe_pool_get_index_locked(struct rxe_pool *pool, u32 index)break ; } -if (node) { -kref_get (&elem->ref_cnt); +if (node &&kref_get_unless_zero (&elem->ref_cnt)) obj = elem->obj; - }else { - obj =NULL ; - }return obj; }
@@ -443,7 +460,7 @@ void *rxe_pool_get_key_locked(struct rxe_pool *pool, void *key) {struct rb_node *node;struct rxe_pool_entry *elem; -void *obj; +void *obj =NULL ;int cmp; node = pool->key.tree.rb_node;
@@ -462,12 +479,8 @@ void *rxe_pool_get_key_locked(struct rxe_pool *pool, void *key)break ; } -if (node) { -kref_get (&elem->ref_cnt); +if (node &&kref_get_unless_zero (&elem->ref_cnt)) obj = elem->obj; - }else { - obj =NULL ; - }return obj; }
drivers/infiniband/sw/rxe/rxe_pool.h +13 -2
@@ -131,9 +131,20 @@ void *rxe_pool_get_key(struct rxe_pool *pool, void *key);void rxe_elem_release (struct kref *kref);/* take a reference on an object */ -#define rxe_add_ref (elem) kref_get(&(elem)->pelem.ref_cnt) +static inline int __rxe_add_ref (struct rxe_pool_entry *elem) +{ +int ret =kref_get_unless_zero (&elem->ref_cnt); + +if (unlikely (!ret)) +pr_warn ("Taking a reference on a %s object that is already destroyed\n" , + elem->pool->name); + +return (ret) ?0 : -EINVAL ; +} + +#define rxe_add_ref (obj) __rxe_add_ref(&(obj)->pelem)/* drop a reference on an object */ -#define rxe_drop_ref (elem) kref_put(&(elem)->pelem.ref_cnt, rxe_elem_release) +#define rxe_drop_ref (obj) kref_put(&(obj)->pelem.ref_cnt, rxe_elem_release)#endif /* RXE_POOL_H */
Signature · 1 line
2.30.2
PATCH v2 7/10 RDMA/rxe: Separate out last rxe_drop_ref
In reply to Bob PearsonCurrently rxe_drop_ref() can be called from a destroy or deallocate verb or from someone releasing a reference protecting a pointer. This leads to the possibility that the object will be destroyed after rdma-core thinks it has. This was the original intent of using kref's but can cause other problems.
This patch separates out the last rxe_drop_ref as rxe_fini_ref() which can return an error if the object is not ready to be destroyed and changes rxe_drop_ref() to return an error if the object has already been destroyed. Now the destroy verbs will return -EBUSY if the object cannot be destroyed and other add/drop references will return -EINVAL if the object has already or would be destroyed. Correct programs should not normally return these values.
Locking is added so that cleanup is executed atomically. Some drop references have their order changed so references are dropped after the pointer they protect has been successfully removed.
This patch exposes some referencing errors which are addressed in the following patches.
--- drivers/infiniband/sw/rxe/rxe_cq.c | 9 +- drivers/infiniband/sw/rxe/rxe_loc.h | 3 +- drivers/infiniband/sw/rxe/rxe_mr.c | 10 +- drivers/infiniband/sw/rxe/rxe_mw.c | 23 ++-- drivers/infiniband/sw/rxe/rxe_pool.c | 148 ++++++++++++++++++++------ drivers/infiniband/sw/rxe/rxe_pool.h | 37 ++++--- drivers/infiniband/sw/rxe/rxe_qp.c | 1 + drivers/infiniband/sw/rxe/rxe_srq.c | 8 ++ drivers/infiniband/sw/rxe/rxe_verbs.c | 40 ++++--- 9 files changed, 192 insertions(+), 87 deletions(-)
drivers/infiniband/sw/rxe/rxe_cq.c +3 -6
@@ -141,18 +141,15 @@ int rxe_cq_post(struct rxe_cq *cq, struct rxe_cqe *cqe, int solicited)return 0 ; } -void rxe_cq_disable (struct rxe_cq *cq) +void rxe_cq_cleanup (struct rxe_pool_entry *arg) { +struct rxe_cq *cq =container_of (arg,typeof (*cq), pelem);unsigned long flags; +/* TODO get rid of this */ spin_lock_irqsave (&cq->cq_lock, flags); cq->is_dying = true;spin_unlock_irqrestore (&cq->cq_lock, flags); -} - -void rxe_cq_cleanup (struct rxe_pool_entry *arg) -{ -struct rxe_cq *cq =container_of (arg,typeof (*cq), pelem);if (cq->queue)rxe_queue_cleanup (cq->queue);
drivers/infiniband/sw/rxe/rxe_loc.h +1 -2
@@ -35,8 +35,6 @@ int rxe_cq_resize_queue(struct rxe_cq *cq, int new_cqe,int rxe_cq_post (struct rxe_cq *cq,struct rxe_cqe *cqe,int solicited); -void rxe_cq_disable (struct rxe_cq *cq); -void rxe_cq_cleanup (struct rxe_pool_entry *arg);/* rxe_mcast.c */
@@ -187,6 +185,7 @@ int rxe_srq_from_init(struct rxe_dev *rxe, struct rxe_srq *srq,int rxe_srq_from_attr (struct rxe_dev *rxe,struct rxe_srq *srq,struct ib_srq_attr *attr,enum ib_srq_attr_mask mask,struct rxe_modify_srq_cmd *ucmd,struct ib_udata *udata); +void rxe_srq_cleanup (struct rxe_pool_entry *arg);void rxe_dealloc (struct ib_device *ib_dev);
drivers/infiniband/sw/rxe/rxe_mr.c +8 -2
@@ -684,6 +684,8 @@ int rxe_mr_set_page(struct ib_mr *ibmr, u64 addr)int rxe_dereg_mr (struct ib_mr *ibmr,struct ib_udata *udata) {struct rxe_mr *mr =to_rmr (ibmr); +struct rxe_pd *pd =mr_pd (mr); +int err;if (atomic_read (&mr->num_mw) >0 ) { pr_warn("%s: Attempt to deregister an MR while bound to MWs\n",
@@ -692,8 +694,12 @@ int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) } mr->state =RXE_MR_STATE_INVALID ; -rxe_drop_ref (mr_pd (mr)); -rxe_drop_ref (mr); + + err =rxe_fini_ref (mr); +if (err) +return err; + +rxe_drop_ref (pd);return 0 ; }
drivers/infiniband/sw/rxe/rxe_mw.c +12 -11
@@ -56,12 +56,15 @@ int rxe_dealloc_mw(struct ib_mw *ibmw)struct rxe_mw *mw =to_rmw (ibmw);struct rxe_pd *pd =to_rpd (ibmw->pd);unsigned long flags; +int err;spin_lock_irqsave (&mw->lock, flags);rxe_do_dealloc_mw (mw);spin_unlock_irqrestore (&mw->lock, flags); -rxe_drop_ref (mw); + err =rxe_fini_ref (mw); +if (err) +return err;rxe_drop_ref (pd);return 0 ;
@@ -177,9 +180,9 @@ static void rxe_do_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe, }if (mw->length) { +/* take over ref on mr from caller */ mw->mr = mr;atomic_inc (&mr->num_mw); -rxe_add_ref (mr); }if (mw->ibmw.type ==IB_MW_TYPE_2 ) {
@@ -192,7 +195,7 @@ int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe) {int ret;struct rxe_mw *mw; -struct rxe_mr *mr; +struct rxe_mr *mr =NULL ;struct rxe_dev *rxe =to_rdev (qp->ibqp.device);u32 mw_rkey = wqe->wr.wr.mw.mw_rkey;u32 mr_lkey = wqe->wr.wr.mw.mr_lkey;
@@ -217,25 +220,23 @@ int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe) }if (unlikely (mr->lkey != mr_lkey)) { +rxe_drop_ref (mr); ret = -EINVAL ; - goto err_drop_mr; + goto err_drop_mw; } - }else { - mr =NULL ; }spin_lock_irqsave (&mw->lock, flags); - ret =rxe_check_bind_mw (qp, wqe, mw, mr); -if (ret) +if (ret) { +if (mr) +rxe_drop_ref (mr); goto err_unlock; + }rxe_do_bind_mw (qp, wqe, mw, mr); err_unlock:spin_unlock_irqrestore (&mw->lock, flags); -err_drop_mr: -if (mr) -rxe_drop_ref (mr); err_drop_mw:rxe_drop_ref (mw); err:
drivers/infiniband/sw/rxe/rxe_pool.c +117 -31
199 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_pool.h +24 -13
@@ -7,6 +7,8 @@#ifndef RXE_POOL_H #define RXE_POOL_H +#include <linux/refcount.h> +#define RXE_POOL_ALIGN (16)#define RXE_POOL_CACHE_FLAGS (0)
@@ -47,7 +49,7 @@ struct rxe_type_info {struct rxe_pool_entry {struct rxe_pool *pool;void *obj; -struct kref ref_cnt; +refcount_t refcnt;struct list_head list;/* only used if keyed */
@@ -127,24 +129,33 @@ void *rxe_pool_get_key_locked(struct rxe_pool *pool, void *key);void *rxe_pool_get_key (struct rxe_pool *pool,void *key); -/* cleanup an object when all references are dropped */ -void rxe_elem_release (struct kref *kref); -/* take a reference on an object */ -static inline int __rxe_add_ref (struct rxe_pool_entry *elem) -{ -int ret =kref_get_unless_zero (&elem->ref_cnt); +int __rxe_add_ref_locked (struct rxe_pool_entry *elem); -if (unlikely (!ret)) -pr_warn ("Taking a reference on a %s object that is already destroyed\n" , - elem->pool->name); +#define rxe_add_ref_locked (obj) __rxe_add_ref_locked(&(obj)->pelem) -return (ret) ?0 : -EINVAL ; -} +int __rxe_add_ref (struct rxe_pool_entry *elem);#define rxe_add_ref (obj) __rxe_add_ref(&(obj)->pelem)/* drop a reference on an object */ -#define rxe_drop_ref (obj) kref_put(&(obj)->pelem.ref_cnt, rxe_elem_release) +int __rxe_drop_ref_locked (struct rxe_pool_entry *elem); + +#define rxe_drop_ref_locked (obj) __rxe_drop_ref_locked(&(obj)->pelem) + +int __rxe_drop_ref (struct rxe_pool_entry *elem); + +#define rxe_drop_ref (obj) __rxe_drop_ref(&(obj)->pelem) + +/* drop last reference on an object */ +int __rxe_fini_ref_locked (struct rxe_pool_entry *elem); + +#define rxe_fini_ref_locked (obj) __rxe_fini_ref_locked(&(obj)->pelem) + +int __rxe_fini_ref (struct rxe_pool_entry *elem); + +#define rxe_fini_ref (obj) __rxe_fini_ref(&(obj)->pelem) + +#define rxe_read_ref (obj) refcount_read(&(obj)->pelem.refcnt)#endif /* RXE_POOL_H */
drivers/infiniband/sw/rxe/rxe_qp.c +1 -0
@@ -835,5 +835,6 @@ void rxe_qp_cleanup(struct rxe_pool_entry *arg) {struct rxe_qp *qp =container_of (arg,typeof (*qp), pelem); +rxe_qp_destroy (qp);execute_in_process_context (rxe_qp_do_cleanup, &qp->cleanup_work); }
drivers/infiniband/sw/rxe/rxe_srq.c +8 -0
@@ -154,3 +154,11 @@ int rxe_srq_from_attr(struct rxe_dev *rxe, struct rxe_srq *srq, srq->rq.queue =NULL ;return err; } + +void rxe_srq_cleanup (struct rxe_pool_entry *arg) +{ +struct rxe_srq *srq =container_of (arg,typeof (*srq), pelem); + +if (srq->rq.queue) +rxe_queue_cleanup (srq->rq.queue); +}
drivers/infiniband/sw/rxe/rxe_verbs.c +18 -22
@@ -115,7 +115,7 @@ static void rxe_dealloc_ucontext(struct ib_ucontext *ibuc) {struct rxe_ucontext *uc =to_ruc (ibuc); -rxe_drop_ref (uc); +rxe_fini_ref (uc); } static int rxe_port_immutable(struct ib_device *dev, u32 port_num,
@@ -149,8 +149,7 @@ static int rxe_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) {struct rxe_pd *pd =to_rpd (ibpd); -rxe_drop_ref (pd); -return 0 ; +return rxe_fini_ref (pd); } static int rxe_create_ah(struct ib_ah *ibah,
@@ -228,8 +227,7 @@ static int rxe_destroy_ah(struct ib_ah *ibah, u32 flags) {struct rxe_ah *ah =to_rah (ibah); -rxe_drop_ref (ah); -return 0 ; +return rxe_fini_ref (ah); }static int post_one_recv (struct rxe_rq *rq,const struct ib_recv_wr *ibwr)
@@ -313,8 +311,8 @@ static int rxe_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init,return 0 ; err2: +rxe_fini_ref (srq);rxe_drop_ref (pd); -rxe_drop_ref (srq); err1:return err; }
@@ -367,12 +365,15 @@ static int rxe_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)static int rxe_destroy_srq (struct ib_srq *ibsrq,struct ib_udata *udata) {struct rxe_srq *srq =to_rsrq (ibsrq); +struct rxe_pd *pd = srq->pd; +int err; -if (srq->rq.queue) -rxe_queue_cleanup (srq->rq.queue); + err =rxe_fini_ref (srq); +if (err) +return err; + +rxe_drop_ref (pd); -rxe_drop_ref (srq->pd); -rxe_drop_ref (srq);return 0 ; }
@@ -437,12 +438,12 @@ static int rxe_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init, err =rxe_qp_from_init (rxe, qp, pd, init, uresp, ibqp->pd, udata);if (err) - goto qp_init; + goto err_fini;return 0 ; -qp_init: -rxe_drop_ref (qp); +err_fini: +rxe_fini_ref (qp);return err; }
@@ -485,9 +486,7 @@ static int rxe_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) {struct rxe_qp *qp =to_rqp (ibqp); -rxe_qp_destroy (qp); -rxe_drop_ref (qp); -return 0 ; +return rxe_fini_ref (qp); } static int validate_send_wr(struct rxe_qp *qp, const struct ib_send_wr *ibwr,
@@ -797,10 +796,7 @@ static int rxe_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) {struct rxe_cq *cq =to_rcq (ibcq); -rxe_cq_disable (cq); - -rxe_drop_ref (cq); -return 0 ; +return rxe_fini_ref (cq); }static int rxe_resize_cq (struct ib_cq *ibcq,int cqe,struct ib_udata *udata)
@@ -924,8 +920,8 @@ static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd,return &mr->ibmr; err3: +rxe_fini_ref (mr);rxe_drop_ref (pd); -rxe_drop_ref (mr); err2:return ERR_PTR (err); }
@@ -956,8 +952,8 @@ static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,return &mr->ibmr; err2: +rxe_fini_ref (mr);rxe_drop_ref (pd); -rxe_drop_ref (mr); err1:return ERR_PTR (err); }
Signature · 1 line
2.30.2
PATCH v2 8/10 RDMA/rxe: Rewrite rxe_mcast.c
In reply to Bob PearsonRewrite rxe_mcast to support rxe_fini_ref and clean up referencing. Drop mc_elem as rxe pool objects and just use kmalloc/kfree. Take qp references as well as mc_grp references to force all mc groups to be de-attached before destroying QPs. Simplify the api to rxe_verbs.
--- drivers/infiniband/sw/rxe/rxe.c | 8 - drivers/infiniband/sw/rxe/rxe_loc.h | 11 +- drivers/infiniband/sw/rxe/rxe_mcast.c | 205 +++++++++++++++++--------- drivers/infiniband/sw/rxe/rxe_net.c | 22 --- drivers/infiniband/sw/rxe/rxe_pool.c | 6 - drivers/infiniband/sw/rxe/rxe_pool.h | 1 - drivers/infiniband/sw/rxe/rxe_verbs.c | 12 +- drivers/infiniband/sw/rxe/rxe_verbs.h | 3 +- 8 files changed, 143 insertions(+), 125 deletions(-)
drivers/infiniband/sw/rxe/rxe.c +0 -8
@@ -31,7 +31,6 @@ void rxe_dealloc(struct ib_device *ib_dev)rxe_pool_cleanup (&rxe->mr_pool);rxe_pool_cleanup (&rxe->mw_pool);rxe_pool_cleanup (&rxe->mc_grp_pool); -rxe_pool_cleanup (&rxe->mc_elem_pool);if (rxe->tfm)crypto_free_shash (rxe->tfm);
@@ -165,15 +164,8 @@ static int rxe_init_pools(struct rxe_dev *rxe)if (err) goto err9; - err =rxe_pool_init (rxe, &rxe->mc_elem_pool,RXE_TYPE_MC_ELEM , - rxe->attr.max_total_mcast_qp_attach); -if (err) - goto err10; -return 0 ; -err10: -rxe_pool_cleanup (&rxe->mc_grp_pool); err9:rxe_pool_cleanup (&rxe->mw_pool); err8:
drivers/infiniband/sw/rxe/rxe_loc.h +1 -10
@@ -38,19 +38,12 @@ int rxe_cq_post(struct rxe_cq *cq, struct rxe_cqe *cqe, int solicited);void rxe_cq_cleanup (struct rxe_pool_entry *arg);/* rxe_mcast.c */ -int rxe_mcast_get_grp (struct rxe_dev *rxe,union ib_gid *mgid, -struct rxe_mc_grp **grp_p); -int rxe_mcast_add_grp_elem (struct rxe_dev *rxe,struct rxe_qp *qp, -struct rxe_mc_grp *grp); - +union ib_gid *mgid);int rxe_mcast_drop_grp_elem (struct rxe_dev *rxe,struct rxe_qp *qp,union ib_gid *mgid); -void rxe_drop_all_mcast_groups (struct rxe_qp *qp); -void rxe_mc_cleanup (struct rxe_pool_entry *arg); -/* rxe_mmap.c */ struct rxe_mmap_info {struct list_head pending_mmaps;
@@ -104,8 +97,6 @@ int rxe_prepare(struct rxe_pkt_info *pkt, struct sk_buff *skb);int rxe_xmit_packet (struct rxe_qp *qp,struct rxe_pkt_info *pkt,struct sk_buff *skb);const char *rxe_parent_name (struct rxe_dev *rxe,unsigned int port_num); -int rxe_mcast_add (struct rxe_dev *rxe,union ib_gid *mgid); -int rxe_mcast_delete (struct rxe_dev *rxe,union ib_gid *mgid);/* rxe_qp.c */ int rxe_qp_chk_init (struct rxe_dev *rxe,struct ib_qp_init_attr *init);
drivers/infiniband/sw/rxe/rxe_mcast.c +139 -66
296 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_net.c +0 -22
@@ -20,28 +20,6 @@static struct rxe_recv_sockets recv_sockets; -int rxe_mcast_add (struct rxe_dev *rxe,union ib_gid *mgid) -{ -int err; -unsigned char ll_addr[ETH_ALEN ]; - -ipv6_eth_mc_map ((struct in6_addr *)mgid->raw, ll_addr); - err =dev_mc_add (rxe->ndev, ll_addr); - -return err; -} - -int rxe_mcast_delete (struct rxe_dev *rxe,union ib_gid *mgid) -{ -int err; -unsigned char ll_addr[ETH_ALEN ]; - -ipv6_eth_mc_map ((struct in6_addr *)mgid->raw, ll_addr); - err =dev_mc_del (rxe->ndev, ll_addr); - -return err; -} -static struct dst_entry *rxe_find_route4 (struct net_device *ndev,struct in_addr *saddr,struct in_addr *daddr)
drivers/infiniband/sw/rxe/rxe_pool.c +0 -6
@@ -76,16 +76,10 @@ static const struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = { .name = "rxe-mc_grp", .size = sizeof(struct rxe_mc_grp), .elem_offset = offsetof(struct rxe_mc_grp, pelem), - .cleanup = rxe_mc_cleanup, .flags = RXE_POOL_KEY, .key_offset = offsetof(struct rxe_mc_grp, mgid), .key_size = sizeof(union ib_gid), }, - [RXE_TYPE_MC_ELEM] = { - .name = "rxe-mc_elem", - .size = sizeof(struct rxe_mc_elem), - .elem_offset = offsetof(struct rxe_mc_elem, pelem), - }, };static int rxe_pool_init_index (struct rxe_pool *pool,u32 max,u32 min)
drivers/infiniband/sw/rxe/rxe_pool.h +0 -1
@@ -28,7 +28,6 @@ enum rxe_elem_type {RXE_TYPE_MR ,RXE_TYPE_MW ,RXE_TYPE_MC_GRP , -RXE_TYPE_MC_ELEM , RXE_NUM_TYPES, /* keep me last */ };
drivers/infiniband/sw/rxe/rxe_verbs.c +1 -11
@@ -984,20 +984,10 @@ static int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,static int rxe_attach_mcast (struct ib_qp *ibqp,union ib_gid *mgid,u16 mlid) { -int err;struct rxe_dev *rxe =to_rdev (ibqp->device);struct rxe_qp *qp =to_rqp (ibqp); -struct rxe_mc_grp *grp; - -/* takes a ref on grp if successful */ - err =rxe_mcast_get_grp (rxe, mgid, &grp); -if (err) -return err; - - err =rxe_mcast_add_grp_elem (rxe, qp, grp); -rxe_drop_ref (grp); -return err; +return rxe_mcast_add_grp_elem (rxe, qp, mgid); }static int rxe_detach_mcast (struct ib_qp *ibqp,union ib_gid *mgid,u16 mlid)
drivers/infiniband/sw/rxe/rxe_verbs.h +2 -1
@@ -403,7 +403,8 @@ struct rxe_dev {struct rxe_pool mr_pool;struct rxe_pool mw_pool;struct rxe_pool mc_grp_pool; -struct rxe_pool mc_elem_pool; + +atomic_t total_mcast_qp_attach;spinlock_t pending_lock;/* guard pending_mmaps */ struct list_head pending_mmaps;
Signature · 1 line
2.30.2
PATCH v2 9/10 RDMA/rxe: Fix ref error in rxe_av.c
In reply to Bob PearsonThe commit referenced below can take a reference to the AH which is never dropped. This only happens in the UD request path. This patch optionally passes that AH back to the caller so that it can hold the reference while the AV is being accessed and then drop it. Code to do this is added to rxe_req.c. The AV is also passed to rxe_prepare in rxe_net.c as an optimization.
--- drivers/infiniband/sw/rxe/rxe_av.c | 24 ++++++++++-- drivers/infiniband/sw/rxe/rxe_loc.h | 5 ++- drivers/infiniband/sw/rxe/rxe_net.c | 17 +++++---- drivers/infiniband/sw/rxe/rxe_req.c | 55 +++++++++++++++++----------- drivers/infiniband/sw/rxe/rxe_resp.c | 2 +- 5 files changed, 67 insertions(+), 36 deletions(-)
drivers/infiniband/sw/rxe/rxe_av.c +21 -3
@@ -99,11 +99,14 @@ void rxe_av_fill_ip_info(struct rxe_av *av, struct rdma_ah_attr *attr) av->network_type = type; } -struct rxe_av *rxe_get_av (struct rxe_pkt_info *pkt) +struct rxe_av *rxe_get_av (struct rxe_pkt_info *pkt,struct rxe_ah **ahp) {struct rxe_ah *ah;u32 ah_num; +if (ahp) + *ahp =NULL ; +if (!pkt || !pkt->qp)return NULL ;
@@ -117,10 +120,25 @@ struct rxe_av *rxe_get_av(struct rxe_pkt_info *pkt)if (ah_num) {/* only new user provider or kernel client */ ah =rxe_pool_get_index (&pkt->rxe->ah_pool, ah_num); -if (!ah || ah->ah_num != ah_num ||rxe_ah_pd (ah) != pkt->qp->pd) { -pr_warn ("Unable to find AH matching ah_num\n" ); +if (!ah) { +pr_warn ("%s: Unable to find AH matching ah_num\n" , + __func__); +return NULL ; + } + +if (rxe_ah_pd (ah) != pkt->qp->pd) { +pr_warn ("%s: PDs don't match for AH and QP\n" , + __func__); +rxe_drop_ref (ah);return NULL ; } + +/* let caller hold ref to ah */ +if (ahp) + *ahp = ah; +else +rxe_drop_ref (ah); +return &ah->av; }
drivers/infiniband/sw/rxe/rxe_loc.h +3 -2
@@ -19,7 +19,7 @@ void rxe_av_to_attr(struct rxe_av *av, struct rdma_ah_attr *attr);void rxe_av_fill_ip_info (struct rxe_av *av,struct rdma_ah_attr *attr); -struct rxe_av *rxe_get_av (struct rxe_pkt_info *pkt); +struct rxe_av *rxe_get_av (struct rxe_pkt_info *pkt,struct rxe_ah **ahp);/* rxe_cq.c */ int rxe_cq_chk_attr(struct rxe_dev *rxe, struct rxe_cq *cq,
@@ -93,7 +93,8 @@ void rxe_mw_cleanup(struct rxe_pool_entry *arg);/* rxe_net.c */ struct sk_buff *rxe_init_packet (struct rxe_dev *rxe,struct rxe_av *av,int paylen,struct rxe_pkt_info *pkt); -int rxe_prepare (struct rxe_pkt_info *pkt,struct sk_buff *skb); +int rxe_prepare (struct rxe_av *av,struct rxe_pkt_info *pkt, +struct sk_buff *skb);int rxe_xmit_packet (struct rxe_qp *qp,struct rxe_pkt_info *pkt,struct sk_buff *skb);const char *rxe_parent_name (struct rxe_dev *rxe,unsigned int port_num);
drivers/infiniband/sw/rxe/rxe_net.c +9 -8
@@ -271,13 +271,13 @@ static void prepare_ipv6_hdr(struct dst_entry *dst, struct sk_buff *skb, ip6h->payload_len =htons (skb->len -sizeof (*ip6h)); } -static int prepare4 (struct rxe_pkt_info *pkt,struct sk_buff *skb) +static int prepare4 (struct rxe_av *av,struct rxe_pkt_info *pkt, +struct sk_buff *skb) {struct rxe_qp *qp = pkt->qp;struct dst_entry *dst;bool xnet = false;__be16 df =htons (IP_DF ); -struct rxe_av *av =rxe_get_av (pkt);struct in_addr *saddr = &av->sgid_addr._sockaddr_in.sin_addr;struct in_addr *daddr = &av->dgid_addr._sockaddr_in.sin_addr;
@@ -297,11 +297,11 @@ static int prepare4(struct rxe_pkt_info *pkt, struct sk_buff *skb)return 0 ; } -static int prepare6 (struct rxe_pkt_info *pkt,struct sk_buff *skb) +static int prepare6 (struct rxe_av *av,struct rxe_pkt_info *pkt, +struct sk_buff *skb) {struct rxe_qp *qp = pkt->qp;struct dst_entry *dst; -struct rxe_av *av =rxe_get_av (pkt);struct in6_addr *saddr = &av->sgid_addr._sockaddr_in6.sin6_addr;struct in6_addr *daddr = &av->dgid_addr._sockaddr_in6.sin6_addr;
@@ -322,16 +322,17 @@ static int prepare6(struct rxe_pkt_info *pkt, struct sk_buff *skb)return 0 ; } -int rxe_prepare (struct rxe_pkt_info *pkt,struct sk_buff *skb) +int rxe_prepare (struct rxe_av *av,struct rxe_pkt_info *pkt, +struct sk_buff *skb) {int err =0 ;if (skb->protocol ==htons (ETH_P_IP )) - err =prepare4 (pkt, skb); + err =prepare4 (av, pkt, skb);else if (skb->protocol ==htons (ETH_P_IPV6 )) - err =prepare6 (pkt, skb); + err =prepare6 (av, pkt, skb); -if (ether_addr_equal (skb->dev->dev_addr,rxe_get_av (pkt)->dmac)) +if (ether_addr_equal (skb->dev->dev_addr, av->dmac)) pkt->mask |=RXE_LOOPBACK_MASK ;return err;
drivers/infiniband/sw/rxe/rxe_req.c +33 -22
121 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_resp.c +1 -1
@@ -632,7 +632,7 @@ static struct sk_buff *prepare_ack_packet(struct rxe_qp *qp,if (ack->mask &RXE_ATMACK_MASK )atmack_set_orig (ack, qp->resp.atomic_orig); - err =rxe_prepare (ack, skb); + err =rxe_prepare (&qp->pri_av, ack, skb);if (err) {kfree_skb (skb);return NULL ;
Signature · 1 line
2.30.2
PATCH v2 10/10 RDMA/rxe: Replace mr by rkey in responder resources
In reply to Bob PearsonCurrently rxe saves a copy of MR in responder resources for RDMA reads. Since the responder resources are never freed just over written if more are needed this MR may not have a reference freed until the QP is destroyed. This patch uses the rkey instead of the MR and on subsequent packets of a multipacket read reply message it looks up the MR from the rkey for each packet. This makes it possible for a user to deregister an MR or unbind a MW on the fly and get correct behaviour.
--- drivers/infiniband/sw/rxe/rxe_qp.c | 10 +-- drivers/infiniband/sw/rxe/rxe_resp.c | 123 ++++++++++++++++++-------- drivers/infiniband/sw/rxe/rxe_verbs.h | 1 - 3 files changed, 87 insertions(+), 47 deletions(-)
drivers/infiniband/sw/rxe/rxe_qp.c +2 -8
@@ -135,12 +135,8 @@ static void free_rd_atomic_resources(struct rxe_qp *qp)void free_rd_atomic_resource (struct rxe_qp *qp,struct resp_res *res) { -if (res->type ==RXE_ATOMIC_MASK ) { +if (res->type ==RXE_ATOMIC_MASK )kfree_skb (res->atomic.skb); - }else if (res->type ==RXE_READ_MASK ) { -if (res->read.mr) -rxe_drop_ref (res->read.mr); - } res->type =0 ; }
@@ -816,10 +812,8 @@ static void rxe_qp_do_cleanup(struct work_struct *work)if (qp->pd)rxe_drop_ref (qp->pd); -if (qp->resp.mr) { +if (qp->resp.mr)rxe_drop_ref (qp->resp.mr); - qp->resp.mr =NULL ; - }if (qp_type (qp) ==IB_QPT_RC )sk_dst_reset (qp->sk->sk);
drivers/infiniband/sw/rxe/rxe_resp.c +85 -38
154 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -1
@@ -157,7 +157,6 @@ struct resp_res {struct sk_buff *skb; } atomic;struct { -struct rxe_mr *mr;u64 va_org;u32 rkey;u32 length;
Signature · 1 line
2.30.2