Fix memory ordering errors in queues
Subsystems (2)
v3 cover Fix memory ordering errors in queues
These patches optimize the memory ordering in rxe_queue.h so that user space and not kernel space indices are protected for loads with smp_load_acquire() and stores with smp_store_release(). The original implementation of this did not apply to all index references which has recently caused test case errors traced to stale memory loads. These patches fix those errors and also protect kernel indices from malicious modification by user space.
---
v3:
Protected kernel index in shared queues from modification by user space. Pass queue type to allow compiler to optimize queue methods.
v2:
Add a way to distinguish between user and kernel indices.
v1:
Add missing smp_load_acquire() calls.
---
Bob Pearson (3):
RDMA/rxe: Add a type flag to rxe_queue structs
RDMA/rxe: Protect user space index loads/stores
RDMA/rxe: Protext kernel index from user space
drivers/infiniband/sw/rxe/rxe_comp.c | 31 ++- drivers/infiniband/sw/rxe/rxe_cq.c | 32 ++- drivers/infiniband/sw/rxe/rxe_qp.c | 22 ++- drivers/infiniband/sw/rxe/rxe_queue.c | 21 +- drivers/infiniband/sw/rxe/rxe_queue.h | 272 ++++++++++++++++++++------ drivers/infiniband/sw/rxe/rxe_req.c | 46 +++-- drivers/infiniband/sw/rxe/rxe_resp.c | 44 ++++- drivers/infiniband/sw/rxe/rxe_srq.c | 5 +- drivers/infiniband/sw/rxe/rxe_verbs.c | 80 ++++++-- drivers/infiniband/sw/rxe/rxe_verbs.h | 5 +- 10 files changed, 423 insertions(+), 135 deletions(-)
Signature · 1 line
2.30.2
PATCH v3 1/3 RDMA/rxe: Add a type flag to rxe_queue structs
To create optimal code only want to use smp_load_acquire() and smp_store_release() for user indices in rxe_queue APIs since kernel indices are protected by locks which also act as memory barriers. By adding a type to the queues we can determine which indices need to be protected.
--- drivers/infiniband/sw/rxe/rxe_cq.c | 4 +++- drivers/infiniband/sw/rxe/rxe_qp.c | 12 ++++++++---- drivers/infiniband/sw/rxe/rxe_queue.c | 8 ++++---- drivers/infiniband/sw/rxe/rxe_queue.h | 13 ++++++++++--- drivers/infiniband/sw/rxe/rxe_srq.c | 4 +++- 5 files changed, 28 insertions(+), 13 deletions(-)
drivers/infiniband/sw/rxe/rxe_cq.c +3 -1
@@ -59,9 +59,11 @@ int rxe_cq_from_init(struct rxe_dev *rxe, struct rxe_cq *cq, int cqe, struct rxe_create_cq_resp __user *uresp) {int err; +enum queue_type type; + type = uresp ?QUEUE_TYPE_TO_USER :QUEUE_TYPE_KERNEL ; cq->queue =rxe_queue_init (rxe, &cqe, -sizeof (struct rxe_cqe )); +sizeof (struct rxe_cqe ), type);if (!cq->queue) {pr_warn ("unable to create cq\n" );return -ENOMEM ;
drivers/infiniband/sw/rxe/rxe_qp.c +8 -4
@@ -206,6 +206,7 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp, {int err;int wqe_size; +enum queue_type type; err =sock_create_kern (&init_net,AF_INET ,SOCK_DGRAM ,0 , &qp->sk);if (err <0 )
@@ -231,7 +232,9 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp, qp->sq.max_inline = init->cap.max_inline_data = wqe_size; wqe_size +=sizeof (struct rxe_send_wqe ); - qp->sq.queue =rxe_queue_init (rxe, &qp->sq.max_wr, wqe_size); + type = uresp ?QUEUE_TYPE_FROM_USER :QUEUE_TYPE_KERNEL ; + qp->sq.queue =rxe_queue_init (rxe, &qp->sq.max_wr, + wqe_size, type);if (!qp->sq.queue)return -ENOMEM ;
@@ -273,6 +276,7 @@ static int rxe_qp_init_resp(struct rxe_dev *rxe, struct rxe_qp *qp, {int err;int wqe_size; +enum queue_type type;if (!qp->srq) { qp->rq.max_wr = init->cap.max_recv_wr;
@@ -283,9 +287,9 @@ static int rxe_qp_init_resp(struct rxe_dev *rxe, struct rxe_qp *qp,pr_debug ("qp#%d max_wr = %d, max_sge = %d, wqe_size = %d\n" ,qp_num (qp), qp->rq.max_wr, qp->rq.max_sge, wqe_size); - qp->rq.queue =rxe_queue_init (rxe, - &qp->rq.max_wr, - wqe_size); + type = uresp ?QUEUE_TYPE_FROM_USER :QUEUE_TYPE_KERNEL ; + qp->rq.queue =rxe_queue_init (rxe, &qp->rq.max_wr, + wqe_size, type);if (!qp->rq.queue)return -ENOMEM ;
drivers/infiniband/sw/rxe/rxe_queue.c +4 -4
@@ -52,9 +52,8 @@ inline void rxe_queue_reset(struct rxe_queue *q)memset (q->buf->data,0 , q->buf_size -sizeof (struct rxe_queue_buf )); } -struct rxe_queue *rxe_queue_init (struct rxe_dev *rxe, -int *num_elem, -unsigned int elem_size) +struct rxe_queue *rxe_queue_init (struct rxe_dev *rxe,int *num_elem, +unsigned int elem_size,enum queue_type type) {struct rxe_queue *q;size_t buf_size;
@@ -69,6 +68,7 @@ struct rxe_queue *rxe_queue_init(struct rxe_dev *rxe, goto err1; q->rxe = rxe; + q->type = type;/* used in resize, only need to copy used part of queue */ q->elem_size = elem_size;
@@ -136,7 +136,7 @@ int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p,int err;unsigned long flags =0 , flags1; - new_q =rxe_queue_init (q->rxe, &num_elem, elem_size); + new_q =rxe_queue_init (q->rxe, &num_elem, elem_size, q->type);if (!new_q)return -ENOMEM ;
drivers/infiniband/sw/rxe/rxe_queue.h +10 -3
@@ -19,6 +19,13 @@* of the queue is one less than the number of element slots */ +/* type of queue */ +enum queue_type { +QUEUE_TYPE_KERNEL , +QUEUE_TYPE_TO_USER , +QUEUE_TYPE_FROM_USER , +}; +struct rxe_queue {struct rxe_dev *rxe;struct rxe_queue_buf *buf;
@@ -27,6 +34,7 @@ struct rxe_queue {size_t elem_size;unsigned int log2_elem_size;u32 index_mask; +enum queue_type type; }; int do_mmap_info(struct rxe_dev *rxe, struct mminfo __user *outbuf,
@@ -35,9 +43,8 @@ int do_mmap_info(struct rxe_dev *rxe, struct mminfo __user *outbuf,void rxe_queue_reset (struct rxe_queue *q); -struct rxe_queue *rxe_queue_init (struct rxe_dev *rxe, -int *num_elem, -unsigned int elem_size); +struct rxe_queue *rxe_queue_init (struct rxe_dev *rxe,int *num_elem, +unsigned int elem_size,enum queue_type type); int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p, unsigned int elem_size, struct ib_udata *udata,
drivers/infiniband/sw/rxe/rxe_srq.c +3 -1
@@ -78,6 +78,7 @@ int rxe_srq_from_init(struct rxe_dev *rxe, struct rxe_srq *srq,int err;int srq_wqe_size;struct rxe_queue *q; +enum queue_type type; srq->ibsrq.event_handler = init->event_handler; srq->ibsrq.srq_context = init->srq_context;
@@ -91,8 +92,9 @@ int rxe_srq_from_init(struct rxe_dev *rxe, struct rxe_srq *srq,spin_lock_init (&srq->rq.producer_lock);spin_lock_init (&srq->rq.consumer_lock); + type = uresp ?QUEUE_TYPE_FROM_USER :QUEUE_TYPE_KERNEL ; q =rxe_queue_init (rxe, &srq->rq.max_wr, - srq_wqe_size); + srq_wqe_size, type);if (!q) {pr_warn ("unable to allocate queue for srq\n" );return -ENOMEM ;
Signature · 1 line
2.30.2
Zhu Yanjun
maintainerRe: [PATCH for-next v3 1/3] RDMA/rxe: Add a type flag to rxe_queue structs
Quoted 21 lines · Bob Pearson
To create optimal code only want to use
smp_load_acquire()andsmp_store_release()for user indices inrxe_queueAPIs since kernel indices are protected by locks which also act as memory barriers. By adding a type to the queues we can determine which indices need to be protected.Signed-off-by Bob Pearson <[email protected]>--- drivers/infiniband/sw/rxe/rxe_cq.c | 4 +++- drivers/infiniband/sw/rxe/rxe_qp.c | 12 ++++++++---- drivers/infiniband/sw/rxe/rxe_queue.c | 8 ++++---- drivers/infiniband/sw/rxe/rxe_queue.h | 13 ++++++++++--- drivers/infiniband/sw/rxe/rxe_srq.c | 4 +++- 5 files changed, 28 insertions(+), 13 deletions(-)drivers/infiniband/sw/rxe/rxe_cq.c +0
-0
drivers/infiniband/sw/rxe/rxe_cq.c int rxe_cq_from_init(struct rxe_dev *rxe, struct rxe_cq *cq, int cqe, · line 64
Quoted 6 lines
drivers/infiniband/sw/rxe/rxe_cq.c +2
-0@@ -59,9 +59,11 @@ int rxe_cq_from_init(struct rxe_dev *rxe, struct rxe_cq *cq, int cqe, struct rxe_create_cq_resp __user *uresp) {int err; +enum queue_type type; + type = uresp ?QUEUE_TYPE_TO_USER :QUEUE_TYPE_KERNEL ;
When is QUEUE_TYPE_TO_USER used? and when is QUEUE_TYPE_FROM_USER is used?
Zhu Yanjun
Quoted 147 lines
drivers/infiniband/sw/rxe/rxe_cq.c +1
-1cq->queue =rxe_queue_init (rxe, &cqe, -sizeof (struct rxe_cqe )); +sizeof (struct rxe_cqe ), type);if (!cq->queue) {pr_warn ("unable to create cq\n" );return -ENOMEM ;drivers/infiniband/sw/rxe/rxe_qp.c +8
-4@@ -206,6 +206,7 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp, {int err;int wqe_size; +enum queue_type type; err =sock_create_kern (&init_net,AF_INET ,SOCK_DGRAM ,0 , &qp->sk);if (err <0 )@@ -231,7 +232,9 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp, qp->sq.max_inline = init->cap.max_inline_data = wqe_size; wqe_size +=sizeof (struct rxe_send_wqe ); - qp->sq.queue =rxe_queue_init (rxe, &qp->sq.max_wr, wqe_size); + type = uresp ?QUEUE_TYPE_FROM_USER :QUEUE_TYPE_KERNEL ; + qp->sq.queue =rxe_queue_init (rxe, &qp->sq.max_wr, + wqe_size, type);if (!qp->sq.queue)return -ENOMEM ;@@ -273,6 +276,7 @@ static int rxe_qp_init_resp(struct rxe_dev *rxe, struct rxe_qp *qp, {int err;int wqe_size; +enum queue_type type;if (!qp->srq) { qp->rq.max_wr = init->cap.max_recv_wr;@@ -283,9 +287,9 @@ static int rxe_qp_init_resp(struct rxe_dev *rxe, struct rxe_qp *qp,pr_debug ("qp#%d max_wr = %d, max_sge = %d, wqe_size = %d\n" ,qp_num (qp), qp->rq.max_wr, qp->rq.max_sge, wqe_size); - qp->rq.queue =rxe_queue_init (rxe, - &qp->rq.max_wr, - wqe_size); + type = uresp ?QUEUE_TYPE_FROM_USER :QUEUE_TYPE_KERNEL ; + qp->rq.queue =rxe_queue_init (rxe, &qp->rq.max_wr, + wqe_size, type);if (!qp->rq.queue)return -ENOMEM ;drivers/infiniband/sw/rxe/rxe_queue.c +4
-4@@ -52,9 +52,8 @@ inline void rxe_queue_reset(struct rxe_queue *q)memset (q->buf->data,0 , q->buf_size -sizeof (struct rxe_queue_buf )); } -struct rxe_queue *rxe_queue_init (struct rxe_dev *rxe, -int *num_elem, -unsigned int elem_size) +struct rxe_queue *rxe_queue_init (struct rxe_dev *rxe,int *num_elem, +unsigned int elem_size,enum queue_type type) {struct rxe_queue *q;size_t buf_size;@@ -69,6 +68,7 @@ struct rxe_queue *rxe_queue_init(struct rxe_dev *rxe, goto err1; q->rxe = rxe; + q->type = type;/* used in resize, only need to copy used part of queue */ q->elem_size = elem_size;@@ -136,7 +136,7 @@ int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p,int err;unsigned long flags =0 , flags1; - new_q =rxe_queue_init (q->rxe, &num_elem, elem_size); + new_q =rxe_queue_init (q->rxe, &num_elem, elem_size, q->type);if (!new_q)return -ENOMEM ;drivers/infiniband/sw/rxe/rxe_queue.h +10
-3@@ -19,6 +19,13 @@* of the queue is one less than the number of element slots */ +/* type of queue */ +enum queue_type { +QUEUE_TYPE_KERNEL , +QUEUE_TYPE_TO_USER , +QUEUE_TYPE_FROM_USER , +}; +struct rxe_queue {struct rxe_dev *rxe;struct rxe_queue_buf *buf;@@ -27,6 +34,7 @@ struct rxe_queue {size_t elem_size;unsigned int log2_elem_size;u32 index_mask; +enum queue_type type; }; int do_mmap_info(struct rxe_dev *rxe, struct mminfo __user *outbuf,@@ -35,9 +43,8 @@ int do_mmap_info(struct rxe_dev *rxe, struct mminfo __user *outbuf,void rxe_queue_reset (struct rxe_queue *q); -struct rxe_queue *rxe_queue_init (struct rxe_dev *rxe, -int *num_elem, -unsigned int elem_size); +struct rxe_queue *rxe_queue_init (struct rxe_dev *rxe,int *num_elem, +unsigned int elem_size,enum queue_type type); int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p, unsigned int elem_size, struct ib_udata *udata,drivers/infiniband/sw/rxe/rxe_srq.c +3
-2@@ -78,6 +78,7 @@ int rxe_srq_from_init(struct rxe_dev *rxe, struct rxe_srq *srq,int err;int srq_wqe_size;struct rxe_queue *q; +enum queue_type type; srq->ibsrq.event_handler = init->event_handler; srq->ibsrq.srq_context = init->srq_context;@@ -91,8 +92,9 @@ int rxe_srq_from_init(struct rxe_dev *rxe, struct rxe_srq *srq,spin_lock_init (&srq->rq.producer_lock);spin_lock_init (&srq->rq.consumer_lock); + type = uresp ?QUEUE_TYPE_FROM_USER :QUEUE_TYPE_KERNEL ; q =rxe_queue_init (rxe, &srq->rq.max_wr, - srq_wqe_size); + srq_wqe_size, type);if (!q) {pr_warn ("unable to allocate queue for srq\n" );return -ENOMEM ; --2.30.2
Re: [PATCH for-next v3 1/3] RDMA/rxe: Add a type flag to rxe_queue structs
Quoted 26 lines · Zhu Yanjun
Quoted 25 lines · Bob Pearson
To create optimal code only want to use
smp_load_acquire()andsmp_store_release()for user indices inrxe_queueAPIs since kernel indices are protected by locks which also act as memory barriers. By adding a type to the queues we can determine which indices need to be protected.Signed-off-by Bob Pearson <[email protected]>--- drivers/infiniband/sw/rxe/rxe_cq.c | 4 +++- drivers/infiniband/sw/rxe/rxe_qp.c | 12 ++++++++---- drivers/infiniband/sw/rxe/rxe_queue.c | 8 ++++---- drivers/infiniband/sw/rxe/rxe_queue.h | 13 ++++++++++--- drivers/infiniband/sw/rxe/rxe_srq.c | 4 +++- 5 files changed, 28 insertions(+), 13 deletions(-)drivers/infiniband/sw/rxe/rxe_cq.c +1
-0@@ -59,9 +59,11 @@ int rxe_cq_from_init(struct rxe_dev *rxe, struct rxe_cq *cq, int cqe, struct rxe_create_cq_resp __user *uresp) {int err; +enum queue_type type;
Quoted 4 lines
drivers/infiniband/sw/rxe/rxe_cq.c int rxe_cq_from_init(struct rxe_dev *rxe, struct rxe_cq *cq, int cqe, · line 64
Quoted 1 line
drivers/infiniband/sw/rxe/rxe_cq.c +1
-0+ type = uresp ?QUEUE_TYPE_TO_USER :QUEUE_TYPE_KERNEL ;When is
QUEUE_TYPE_TO_USERused? and when isQUEUE_TYPE_FROM_USERis used?Zhu Yanjun
QUEUE_TYPE_FROM_USER is used for user space send, recv and shared receive work queues. QUEUE_TYPE_TO_USER is used for user space completion queues. QUEUE_TYPE_KERNEL is used for kernel verbs consumers for any queue.
Bob
Quoted 1 line
PATCH v3 2/3 RDMA/rxe: Protect user space index loads/stores
In reply to Bob PearsonModify the queue APIs to protect all user space index loads with smp_load_acquire() and all user space index stores with smp_store_release(). Base this on the types of the queues which can be one of ..KERNEL, ..FROM_USER, ..TO_USER. Kernel space indices are protected by locks which also provide memory barriers.
--- drivers/infiniband/sw/rxe/rxe_queue.h | 168 ++++++++++++++++++-------- 1 file changed, 117 insertions(+), 51 deletions(-)
drivers/infiniband/sw/rxe/rxe_queue.h +117 -51
@@ -66,12 +66,22 @@ static inline int queue_empty(struct rxe_queue *q)u32 prod;u32 cons; -/* make sure all changes to queue complete before -* testing queue empty -*/ - prod =smp_load_acquire (&q->buf->producer_index); -/* same */ - cons =smp_load_acquire (&q->buf->consumer_index); +switch (q->type) { +case QUEUE_TYPE_FROM_USER : +/* protect user space index */ + prod =smp_load_acquire (&q->buf->producer_index); + cons = q->buf->consumer_index; +break ; +case QUEUE_TYPE_TO_USER : + prod = q->buf->producer_index; +/* protect user space index */ + cons =smp_load_acquire (&q->buf->consumer_index); +break ; +case QUEUE_TYPE_KERNEL : + prod = q->buf->producer_index; + cons = q->buf->consumer_index; +break ; + }return ((prod - cons) & q->index_mask) ==0 ; }
@@ -81,95 +91,151 @@ static inline int queue_full(struct rxe_queue *q)u32 prod;u32 cons; -/* make sure all changes to queue complete before -* testing queue full -*/ - prod =smp_load_acquire (&q->buf->producer_index); -/* same */ - cons =smp_load_acquire (&q->buf->consumer_index); +switch (q->type) { +case QUEUE_TYPE_FROM_USER : +/* protect user space index */ + prod =smp_load_acquire (&q->buf->producer_index); + cons = q->buf->consumer_index; +break ; +case QUEUE_TYPE_TO_USER : + prod = q->buf->producer_index; +/* protect user space index */ + cons =smp_load_acquire (&q->buf->consumer_index); +break ; +case QUEUE_TYPE_KERNEL : + prod = q->buf->producer_index; + cons = q->buf->consumer_index; +break ; + }return ((prod +1 - cons) & q->index_mask) ==0 ; } -static inline void advance_producer (struct rxe_queue *q) +static inline unsigned int queue_count (const struct rxe_queue *q) {u32 prod; +u32 cons; - prod = (q->buf->producer_index +1 ) & q->index_mask; +switch (q->type) { +case QUEUE_TYPE_FROM_USER : +/* protect user space index */ + prod =smp_load_acquire (&q->buf->producer_index); + cons = q->buf->consumer_index; +break ; +case QUEUE_TYPE_TO_USER : + prod = q->buf->producer_index; +/* protect user space index */ + cons =smp_load_acquire (&q->buf->consumer_index); +break ; +case QUEUE_TYPE_KERNEL : + prod = q->buf->producer_index; + cons = q->buf->consumer_index; +break ; + } + +return (prod - cons) & q->index_mask; +} + +static inline void advance_producer (struct rxe_queue *q) +{ +u32 prod; -/* make sure all changes to queue complete before -* changing producer index -*/ -smp_store_release (&q->buf->producer_index, prod); +if (q->type ==QUEUE_TYPE_FROM_USER ) { +/* protect user space index */ + prod =smp_load_acquire (&q->buf->producer_index); + prod = (prod +1 ) & q->index_mask; +/* same */ +smp_store_release (&q->buf->producer_index, prod); + }else { + prod = q->buf->producer_index; + q->buf->producer_index = (prod +1 ) & q->index_mask; + } }static inline void advance_consumer (struct rxe_queue *q) {u32 cons; - cons = (q->buf->consumer_index +1 ) & q->index_mask; - -/* make sure all changes to queue complete before -* changing consumer index -*/ -smp_store_release (&q->buf->consumer_index, cons); +if (q->type ==QUEUE_TYPE_TO_USER ) { +/* protect user space index */ + cons =smp_load_acquire (&q->buf->consumer_index); + cons = (cons +1 ) & q->index_mask; +/* same */ +smp_store_release (&q->buf->consumer_index, cons); + }else { + cons = q->buf->consumer_index; + q->buf->consumer_index = (cons +1 ) & q->index_mask; + } }static inline void *producer_addr (struct rxe_queue *q) { -return q->buf->data + ((q->buf->producer_index & q->index_mask) - << q->log2_elem_size); +u32 prod; + +if (q->type ==QUEUE_TYPE_FROM_USER ) +/* protect user space index */ + prod =smp_load_acquire (&q->buf->producer_index); +else + prod = q->buf->producer_index; + +return q->buf->data + ((prod & q->index_mask) << q->log2_elem_size); }static inline void *consumer_addr (struct rxe_queue *q) { -return q->buf->data + ((q->buf->consumer_index & q->index_mask) - << q->log2_elem_size); +u32 cons; + +if (q->type ==QUEUE_TYPE_TO_USER ) +/* protect user space index */ + cons =smp_load_acquire (&q->buf->consumer_index); +else + cons = q->buf->consumer_index; + +return q->buf->data + ((cons & q->index_mask) << q->log2_elem_size); }static inline unsigned int producer_index (struct rxe_queue *q) { -u32 index; +u32 prod; + +if (q->type ==QUEUE_TYPE_FROM_USER ) +/* protect user space index */ + prod =smp_load_acquire (&q->buf->producer_index); +else + prod = q->buf->producer_index; -/* make sure all changes to queue -* complete before getting producer index -*/ - index =smp_load_acquire (&q->buf->producer_index); - index &= q->index_mask; + prod &= q->index_mask; -return index; +return prod; }static inline unsigned int consumer_index (struct rxe_queue *q) { -u32 index; +u32 cons; -/* make sure all changes to queue -* complete before getting consumer index -*/ - index =smp_load_acquire (&q->buf->consumer_index); - index &= q->index_mask; +if (q->type ==QUEUE_TYPE_TO_USER ) +/* protect user space index */ + cons =smp_load_acquire (&q->buf->consumer_index); +else + cons = q->buf->consumer_index; -return index; + cons &= q->index_mask; + +return cons; } -static inline void *addr_from_index (struct rxe_queue *q,unsigned int index) +static inline void *addr_from_index (struct rxe_queue *q, +unsigned int index) {return q->buf->data + ((index & q->index_mask) << q->buf->log2_elem_size); }static inline unsigned int index_from_addr (const struct rxe_queue *q, -const void *addr) +const void *addr) {return (((u8 *)addr - q->buf->data) >> q->log2_elem_size) - & q->index_mask; -} - -static inline unsigned int queue_count (const struct rxe_queue *q) -{ -return (q->buf->producer_index - q->buf->consumer_index) - & q->index_mask; + & q->index_mask; }static inline void *queue_head (struct rxe_queue *q)
Signature · 1 line
2.30.2
PATCH v3 3/3 RDMA/rxe: Protext kernel index from user space
In reply to Bob PearsonIn order to prevent user space from modifying the index that belongs to the kernel for shared queues let the kernel use a local copy of the index and copy any new values of that index to the shared rxe_queue_bus struct.
This adds more switch statements which decrease the peformance of the queue API. Move the type into the parameter list for these functions so that the compiler can optimize out the switch statements when the explicit type is known. Modify all the calls in the driver on performance paths to pass in the explicit queue type.
--- drivers/infiniband/sw/rxe/rxe_comp.c | 31 ++++-- drivers/infiniband/sw/rxe/rxe_cq.c | 28 ++++- drivers/infiniband/sw/rxe/rxe_qp.c | 10 +- drivers/infiniband/sw/rxe/rxe_queue.c | 13 +-- drivers/infiniband/sw/rxe/rxe_queue.h | 145 +++++++++++++++++++------- drivers/infiniband/sw/rxe/rxe_req.c | 46 ++++++-- drivers/infiniband/sw/rxe/rxe_resp.c | 44 ++++++-- drivers/infiniband/sw/rxe/rxe_srq.c | 1 + drivers/infiniband/sw/rxe/rxe_verbs.c | 80 +++++++++++--- drivers/infiniband/sw/rxe/rxe_verbs.h | 5 +- 10 files changed, 305 insertions(+), 98 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +21 -10
@@ -142,7 +142,10 @@ static inline enum comp_state get_wqe(struct rxe_qp *qp,/* we come here whether or not we found a response packet to see if * there are any posted WQEs */ - wqe =queue_head (qp->sq.queue); +if (qp->is_user) + wqe =queue_head (qp->sq.queue,QUEUE_TYPE_FROM_USER ); +else + wqe =queue_head (qp->sq.queue,QUEUE_TYPE_KERNEL ); *wqe_p = wqe;/* no WQE or requester has not started it yet */
@@ -415,16 +418,23 @@ static void do_complete(struct rxe_qp *qp, struct rxe_send_wqe *wqe) {struct rxe_dev *rxe =to_rdev (qp->ibqp.device);struct rxe_cqe cqe; +bool post; + +/* do we need to post a completion */ + post = ((qp->sq_sig_type ==IB_SIGNAL_ALL_WR ) || + (wqe->wr.send_flags &IB_SEND_SIGNALED ) || + wqe->status !=IB_WC_SUCCESS ); -if ((qp->sq_sig_type ==IB_SIGNAL_ALL_WR ) || - (wqe->wr.send_flags &IB_SEND_SIGNALED ) || - wqe->status !=IB_WC_SUCCESS ) { +if (post)make_send_cqe (qp, wqe, &cqe); -advance_consumer (qp->sq.queue); + +if (qp->is_user) +advance_consumer (qp->sq.queue,QUEUE_TYPE_FROM_USER ); +else +advance_consumer (qp->sq.queue,QUEUE_TYPE_KERNEL ); + +if (post)rxe_cq_post (qp->scq, &cqe,0 ); - }else { -advance_consumer (qp->sq.queue); - } if (wqe->wr.opcode == IB_WR_SEND || wqe->wr.opcode == IB_WR_SEND_WITH_IMM ||
@@ -512,6 +522,7 @@ static void rxe_drain_resp_pkts(struct rxe_qp *qp, bool notify) {struct sk_buff *skb;struct rxe_send_wqe *wqe; +struct rxe_queue *q = qp->sq.queue;while ((skb =skb_dequeue (&qp->resp_pkts))) {rxe_drop_ref (qp);
@@ -519,12 +530,12 @@ static void rxe_drain_resp_pkts(struct rxe_qp *qp, bool notify)ib_device_put (qp->ibqp.device); } -while ((wqe =queue_head (qp->sq.queue))) { +while ((wqe =queue_head (q, q->type))) {if (notify) { wqe->status =IB_WC_WR_FLUSH_ERR ;do_complete (qp, wqe); }else { -advance_consumer (qp->sq.queue); +advance_consumer (q, q->type); } } }
drivers/infiniband/sw/rxe/rxe_cq.c +24 -4
@@ -25,7 +25,11 @@ int rxe_cq_chk_attr(struct rxe_dev *rxe, struct rxe_cq *cq, }if (cq) { - count =queue_count (cq->queue); +if (cq->is_user) + count =queue_count (cq->queue,QUEUE_TYPE_TO_USER ); +else + count =queue_count (cq->queue,QUEUE_TYPE_KERNEL ); +if (cqe < count) {pr_warn ("cqe(%d) < current # elements in queue (%d)" , cqe, count);
@@ -108,10 +112,17 @@ int rxe_cq_post(struct rxe_cq *cq, struct rxe_cqe *cqe, int solicited) {struct ib_event ev;unsigned long flags; +int full; +void *addr;spin_lock_irqsave (&cq->cq_lock, flags); -if (unlikely (queue_full (cq->queue))) { +if (cq->is_user) + full =queue_full (cq->queue,QUEUE_TYPE_TO_USER ); +else + full =queue_full (cq->queue,QUEUE_TYPE_KERNEL ); + +if (unlikely (full)) {spin_unlock_irqrestore (&cq->cq_lock, flags);if (cq->ibcq.event_handler) { ev.device = cq->ibcq.device;
@@ -123,9 +134,18 @@ int rxe_cq_post(struct rxe_cq *cq, struct rxe_cqe *cqe, int solicited)return -EBUSY ; } -memcpy (producer_addr (cq->queue), cqe,sizeof (*cqe)); +if (cq->is_user) + addr =producer_addr (cq->queue,QUEUE_TYPE_TO_USER ); +else + addr =producer_addr (cq->queue,QUEUE_TYPE_KERNEL ); + +memcpy (addr, cqe,sizeof (*cqe)); + +if (cq->is_user) +advance_producer (cq->queue,QUEUE_TYPE_TO_USER ); +else +advance_producer (cq->queue,QUEUE_TYPE_KERNEL ); -advance_producer (cq->queue);spin_unlock_irqrestore (&cq->cq_lock, flags); if ((cq->notify == IB_CQ_NEXT_COMP) ||
drivers/infiniband/sw/rxe/rxe_qp.c +9 -1
@@ -248,7 +248,13 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp,return err; } - qp->req.wqe_index =producer_index (qp->sq.queue); +if (qp->is_user) + qp->req.wqe_index =producer_index (qp->sq.queue, +QUEUE_TYPE_FROM_USER ); +else + qp->req.wqe_index =producer_index (qp->sq.queue, +QUEUE_TYPE_KERNEL ); + qp->req.state =QP_STATE_RESET ; qp->req.opcode =-1 ; qp->comp.opcode =-1 ;
@@ -306,6 +312,8 @@ static int rxe_qp_init_resp(struct rxe_dev *rxe, struct rxe_qp *qp,spin_lock_init (&qp->rq.producer_lock);spin_lock_init (&qp->rq.consumer_lock); + qp->rq.is_user = qp->is_user; +skb_queue_head_init (&qp->resp_pkts); rxe_init_task(rxe, &qp->resp.task, qp,
drivers/infiniband/sw/rxe/rxe_queue.c +7 -6
@@ -111,14 +111,15 @@ struct rxe_queue *rxe_queue_init(struct rxe_dev *rxe, int *num_elem,static int resize_finish (struct rxe_queue *q,struct rxe_queue *new_q,unsigned int num_elem) { -if (!queue_empty (q) && (num_elem <queue_count (q))) +if (!queue_empty (q, q->type) && (num_elem <queue_count (q, q->type)))return -EINVAL ; -while (!queue_empty (q)) { -memcpy (producer_addr (new_q),consumer_addr (q), - new_q->elem_size); -advance_producer (new_q); -advance_consumer (q); +while (!queue_empty (q, q->type)) { +memcpy (producer_addr (new_q, new_q->type), +consumer_addr (q, q->type), + new_q->elem_size); +advance_producer (new_q, new_q->type); +advance_consumer (q, q->type); }swap (*q, *new_q);
drivers/infiniband/sw/rxe/rxe_queue.h +107 -38
275 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_req.c +35 -11
95 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_resp.c +34 -10
93 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_srq.c +1 -0
@@ -86,6 +86,7 @@ int rxe_srq_from_init(struct rxe_dev *rxe, struct rxe_srq *srq, srq->srq_num = srq->pelem.index; srq->rq.max_wr = init->attr.max_wr; srq->rq.max_sge = init->attr.max_sge; + srq->rq.is_user = srq->is_user; srq_wqe_size =rcv_wqe_size (srq->rq.max_sge);
drivers/infiniband/sw/rxe/rxe_verbs.c +63 -17
157 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_verbs.h +4 -1
@@ -77,6 +77,7 @@ enum wqe_state { };struct rxe_sq { +bool is_user;int max_wr;int max_sge;int max_inline;
@@ -85,6 +86,7 @@ struct rxe_sq { };struct rxe_rq { +bool is_user;int max_wr;int max_sge;spinlock_t producer_lock;/* guard queue producer */
@@ -98,6 +100,7 @@ struct rxe_srq {struct rxe_pd *pd;struct rxe_rq rq;u32 srq_num; +bool is_user;int limit;int error;
@@ -212,7 +215,7 @@ struct rxe_qp {struct ib_qp_attr attr;unsigned int valid;unsigned int mtu; -int is_user; +bool is_user;struct rxe_pd *pd;struct rxe_srq *srq;
Signature · 1 line
2.30.2
Jason Gunthorpe
maintainerQuoted 7 lines · Bob Pearson
These patches optimize the memory ordering in
rxe_queue.hso that user space and not kernel space indices are protected for loads withsmp_load_acquire()and stores withsmp_store_release(). The original implementation of this did not apply to all index references which has recently caused test case errors traced to stale memory loads. These patches fix those errors and also protect kernel indices from malicious modification by user space.
I didn't read it carefully, but I think this captures the basic solution
Jason
Zhu Yanjun
maintainerQuoted 10 lines · Jason Gunthorpe
Quoted 7 lines · Bob Pearson
These patches optimize the memory ordering in
rxe_queue.hso that user space and not kernel space indices are protected for loads withsmp_load_acquire()and stores withsmp_store_release(). The original implementation of this did not apply to all index references which has recently caused test case errors traced to stale memory loads. These patches fix those errors and also protect kernel indices from malicious modification by user space.
Quoted 2 lines
I didn't read it carefully, but I think this captures the basic solution
I made tests with rdma-core.
With or without changes of kernel-headers/rdma/rdma_user_rxe.h, the previously mentioned 17 errors disappear.
So this patch series takes effect.
Zhu Yanjun
Quoted 2 lines
Jason
Quoted 11 lines · Bob Pearson
These patches optimize the memory ordering in
rxe_queue.hso that user space and not kernel space indices are protected for loads withsmp_load_acquire()and stores withsmp_store_release(). The original implementation of this did not apply to all index references which has recently caused test case errors traced to stale memory loads. These patches fix those errors and also protect kernel indices from malicious modification by user space.Reported-by Zhu Yanjun <[email protected]>Fixes d21a1240f516 ("RDMA/rxe: Use acquire/release for memory ordering")Signed-off-by Bob Pearson <[email protected]>
Quoted 13 lines
---
v3:
Protected kernel index in shared queues from modification by user space. Pass queue type to allow compiler to optimize queue methods.
v2:
Add a way to distinguish between user and kernel indices.
v1:
Add missingsmp_load_acquire()calls.
---
Bob Pearson (3):
RDMA/rxe: Add a type flag torxe_queuestructs
RDMA/rxe: Protect user space index loads/stores
RDMA/rxe: Protext kernel index from user space
Applied to for-next, thanks
Jason