Fix memory ordering errors in queues
Subsystems (2)
v2 cover Fix memory ordering errors in queues
These two patches optimize the memory ordering in rxe_queue.h so that user space and not kernel space indices are protected for loads with smp_load_acquire() and stores with smp_store_release(). The original implementation of this did not apply to all index references which has recently caused test case errors traced to stale memory loads. These patches fix those errors.
---
v2:
Added a way to distinguish between user and kernel indices.
---
Bob Pearson (2):
RDMA/rxe: Add a type flag to rxe_queue structs RDMA/rxe: Protect user space index loads/stores
drivers/infiniband/sw/rxe/rxe_cq.c | 4 +- drivers/infiniband/sw/rxe/rxe_qp.c | 12 +- drivers/infiniband/sw/rxe/rxe_queue.c | 8 +- drivers/infiniband/sw/rxe/rxe_queue.h | 181 ++++++++++++++++++-------- drivers/infiniband/sw/rxe/rxe_srq.c | 4 +- 5 files changed, 145 insertions(+), 64 deletions(-)
Signature · 1 line
2.30.2
PATCH v2 1/2 RDMA/rxe: Add a type flag to rxe_queue structs
To create optimal code only want to use smp_load_acquire() and smp_store_release() for user indices in rxe_queue APIs since kernel indices are protected by locks which also act as memory barriers. By adding a type to the queues we can determine which indices need to be protected.
---
v2:
This patch added in v2 to add the type field.
--- drivers/infiniband/sw/rxe/rxe_cq.c | 4 +++- drivers/infiniband/sw/rxe/rxe_qp.c | 12 ++++++++---- drivers/infiniband/sw/rxe/rxe_queue.c | 8 ++++---- drivers/infiniband/sw/rxe/rxe_queue.h | 13 ++++++++++--- drivers/infiniband/sw/rxe/rxe_srq.c | 4 +++- 5 files changed, 28 insertions(+), 13 deletions(-)
drivers/infiniband/sw/rxe/rxe_cq.c +3 -1
@@ -59,9 +59,11 @@ int rxe_cq_from_init(struct rxe_dev *rxe, struct rxe_cq *cq, int cqe, struct rxe_create_cq_resp __user *uresp) {int err; +enum queue_type type; + type = uresp ?QUEUE_TYPE_TO_USER :QUEUE_TYPE_KERNEL ; cq->queue =rxe_queue_init (rxe, &cqe, -sizeof (struct rxe_cqe )); +sizeof (struct rxe_cqe ), type);if (!cq->queue) {pr_warn ("unable to create cq\n" );return -ENOMEM ;
drivers/infiniband/sw/rxe/rxe_qp.c +8 -4
@@ -206,6 +206,7 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp, {int err;int wqe_size; +enum queue_type type; err =sock_create_kern (&init_net,AF_INET ,SOCK_DGRAM ,0 , &qp->sk);if (err <0 )
@@ -231,7 +232,9 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp, qp->sq.max_inline = init->cap.max_inline_data = wqe_size; wqe_size +=sizeof (struct rxe_send_wqe ); - qp->sq.queue =rxe_queue_init (rxe, &qp->sq.max_wr, wqe_size); + type = uresp ?QUEUE_TYPE_FROM_USER :QUEUE_TYPE_KERNEL ; + qp->sq.queue =rxe_queue_init (rxe, &qp->sq.max_wr, + wqe_size, type);if (!qp->sq.queue)return -ENOMEM ;
@@ -273,6 +276,7 @@ static int rxe_qp_init_resp(struct rxe_dev *rxe, struct rxe_qp *qp, {int err;int wqe_size; +enum queue_type type;if (!qp->srq) { qp->rq.max_wr = init->cap.max_recv_wr;
@@ -283,9 +287,9 @@ static int rxe_qp_init_resp(struct rxe_dev *rxe, struct rxe_qp *qp,pr_debug ("qp#%d max_wr = %d, max_sge = %d, wqe_size = %d\n" ,qp_num (qp), qp->rq.max_wr, qp->rq.max_sge, wqe_size); - qp->rq.queue =rxe_queue_init (rxe, - &qp->rq.max_wr, - wqe_size); + type = uresp ?QUEUE_TYPE_FROM_USER :QUEUE_TYPE_KERNEL ; + qp->rq.queue =rxe_queue_init (rxe, &qp->rq.max_wr, + wqe_size, type);if (!qp->rq.queue)return -ENOMEM ;
drivers/infiniband/sw/rxe/rxe_queue.c +4 -4
@@ -52,9 +52,8 @@ inline void rxe_queue_reset(struct rxe_queue *q)memset (q->buf->data,0 , q->buf_size -sizeof (struct rxe_queue_buf )); } -struct rxe_queue *rxe_queue_init (struct rxe_dev *rxe, -int *num_elem, -unsigned int elem_size) +struct rxe_queue *rxe_queue_init (struct rxe_dev *rxe,int *num_elem, +unsigned int elem_size,enum queue_type type) {struct rxe_queue *q;size_t buf_size;
@@ -69,6 +68,7 @@ struct rxe_queue *rxe_queue_init(struct rxe_dev *rxe, goto err1; q->rxe = rxe; + q->type = type;/* used in resize, only need to copy used part of queue */ q->elem_size = elem_size;
@@ -136,7 +136,7 @@ int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p,int err;unsigned long flags =0 , flags1; - new_q =rxe_queue_init (q->rxe, &num_elem, elem_size); + new_q =rxe_queue_init (q->rxe, &num_elem, elem_size, q->type);if (!new_q)return -ENOMEM ;
drivers/infiniband/sw/rxe/rxe_queue.h +10 -3
@@ -19,6 +19,13 @@* of the queue is one less than the number of element slots */ +/* type of queue */ +enum queue_type { +QUEUE_TYPE_KERNEL , +QUEUE_TYPE_TO_USER , +QUEUE_TYPE_FROM_USER , +}; +struct rxe_queue {struct rxe_dev *rxe;struct rxe_queue_buf *buf;
@@ -27,6 +34,7 @@ struct rxe_queue {size_t elem_size;unsigned int log2_elem_size;u32 index_mask; +enum queue_type type; }; int do_mmap_info(struct rxe_dev *rxe, struct mminfo __user *outbuf,
@@ -35,9 +43,8 @@ int do_mmap_info(struct rxe_dev *rxe, struct mminfo __user *outbuf,void rxe_queue_reset (struct rxe_queue *q); -struct rxe_queue *rxe_queue_init (struct rxe_dev *rxe, -int *num_elem, -unsigned int elem_size); +struct rxe_queue *rxe_queue_init (struct rxe_dev *rxe,int *num_elem, +unsigned int elem_size,enum queue_type type); int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p, unsigned int elem_size, struct ib_udata *udata,
drivers/infiniband/sw/rxe/rxe_srq.c +3 -1
@@ -78,6 +78,7 @@ int rxe_srq_from_init(struct rxe_dev *rxe, struct rxe_srq *srq,int err;int srq_wqe_size;struct rxe_queue *q; +enum queue_type type; srq->ibsrq.event_handler = init->event_handler; srq->ibsrq.srq_context = init->srq_context;
@@ -91,8 +92,9 @@ int rxe_srq_from_init(struct rxe_dev *rxe, struct rxe_srq *srq,spin_lock_init (&srq->rq.producer_lock);spin_lock_init (&srq->rq.consumer_lock); + type = uresp ?QUEUE_TYPE_FROM_USER :QUEUE_TYPE_KERNEL ; q =rxe_queue_init (rxe, &srq->rq.max_wr, - srq_wqe_size); + srq_wqe_size, type);if (!q) {pr_warn ("unable to allocate queue for srq\n" );return -ENOMEM ;
Signature · 1 line
2.30.2
PATCH v2 2/2 RDMA/rxe: Protect user space index loads/stores
In reply to Bob PearsonModify the queue APIs to protect all user space index loads with smp_load_acquire() and all user space index stores with smp_store_release(). Base this on the types of the queues which can be one of ..KERNEL, ..FROM_USER, ..TO_USER. Kernel space indices are protected by locks which also provide memory barriers.
---
v2:
In v2 use queue type to selectively protect user space indices.
--- drivers/infiniband/sw/rxe/rxe_queue.h | 168 ++++++++++++++++++-------- 1 file changed, 117 insertions(+), 51 deletions(-)
drivers/infiniband/sw/rxe/rxe_queue.h +117 -51
@@ -66,12 +66,22 @@ static inline int queue_empty(struct rxe_queue *q)u32 prod;u32 cons; -/* make sure all changes to queue complete before -* testing queue empty -*/ - prod =smp_load_acquire (&q->buf->producer_index); -/* same */ - cons =smp_load_acquire (&q->buf->consumer_index); +switch (q->type) { +case QUEUE_TYPE_FROM_USER : +/* protect user space index */ + prod =smp_load_acquire (&q->buf->producer_index); + cons = q->buf->consumer_index; +break ; +case QUEUE_TYPE_TO_USER : + prod = q->buf->producer_index; +/* protect user space index */ + cons =smp_load_acquire (&q->buf->consumer_index); +break ; +case QUEUE_TYPE_KERNEL : + prod = q->buf->producer_index; + cons = q->buf->consumer_index; +break ; + }return ((prod - cons) & q->index_mask) ==0 ; }
@@ -81,95 +91,151 @@ static inline int queue_full(struct rxe_queue *q)u32 prod;u32 cons; -/* make sure all changes to queue complete before -* testing queue full -*/ - prod =smp_load_acquire (&q->buf->producer_index); -/* same */ - cons =smp_load_acquire (&q->buf->consumer_index); +switch (q->type) { +case QUEUE_TYPE_FROM_USER : +/* protect user space index */ + prod =smp_load_acquire (&q->buf->producer_index); + cons = q->buf->consumer_index; +break ; +case QUEUE_TYPE_TO_USER : + prod = q->buf->producer_index; +/* protect user space index */ + cons =smp_load_acquire (&q->buf->consumer_index); +break ; +case QUEUE_TYPE_KERNEL : + prod = q->buf->producer_index; + cons = q->buf->consumer_index; +break ; + }return ((prod +1 - cons) & q->index_mask) ==0 ; } -static inline void advance_producer (struct rxe_queue *q) +static inline unsigned int queue_count (const struct rxe_queue *q) {u32 prod; +u32 cons; - prod = (q->buf->producer_index +1 ) & q->index_mask; +switch (q->type) { +case QUEUE_TYPE_FROM_USER : +/* protect user space index */ + prod =smp_load_acquire (&q->buf->producer_index); + cons = q->buf->consumer_index; +break ; +case QUEUE_TYPE_TO_USER : + prod = q->buf->producer_index; +/* protect user space index */ + cons =smp_load_acquire (&q->buf->consumer_index); +break ; +case QUEUE_TYPE_KERNEL : + prod = q->buf->producer_index; + cons = q->buf->consumer_index; +break ; + } + +return (prod - cons) & q->index_mask; +} + +static inline void advance_producer (struct rxe_queue *q) +{ +u32 prod; -/* make sure all changes to queue complete before -* changing producer index -*/ -smp_store_release (&q->buf->producer_index, prod); +if (q->type ==QUEUE_TYPE_FROM_USER ) { +/* protect user space index */ + prod =smp_load_acquire (&q->buf->producer_index); + prod = (prod +1 ) & q->index_mask; +/* same */ +smp_store_release (&q->buf->producer_index, prod); + }else { + prod = q->buf->producer_index; + q->buf->producer_index = (prod +1 ) & q->index_mask; + } }static inline void advance_consumer (struct rxe_queue *q) {u32 cons; - cons = (q->buf->consumer_index +1 ) & q->index_mask; - -/* make sure all changes to queue complete before -* changing consumer index -*/ -smp_store_release (&q->buf->consumer_index, cons); +if (q->type ==QUEUE_TYPE_TO_USER ) { +/* protect user space index */ + cons =smp_load_acquire (&q->buf->consumer_index); + cons = (cons +1 ) & q->index_mask; +/* same */ +smp_store_release (&q->buf->consumer_index, cons); + }else { + cons = q->buf->consumer_index; + q->buf->consumer_index = (cons +1 ) & q->index_mask; + } }static inline void *producer_addr (struct rxe_queue *q) { -return q->buf->data + ((q->buf->producer_index & q->index_mask) - << q->log2_elem_size); +u32 prod; + +if (q->type ==QUEUE_TYPE_FROM_USER ) +/* protect user space index */ + prod =smp_load_acquire (&q->buf->producer_index); +else + prod = q->buf->producer_index; + +return q->buf->data + ((prod & q->index_mask) << q->log2_elem_size); }static inline void *consumer_addr (struct rxe_queue *q) { -return q->buf->data + ((q->buf->consumer_index & q->index_mask) - << q->log2_elem_size); +u32 cons; + +if (q->type ==QUEUE_TYPE_TO_USER ) +/* protect user space index */ + cons =smp_load_acquire (&q->buf->consumer_index); +else + cons = q->buf->consumer_index; + +return q->buf->data + ((cons & q->index_mask) << q->log2_elem_size); }static inline unsigned int producer_index (struct rxe_queue *q) { -u32 index; +u32 prod; + +if (q->type ==QUEUE_TYPE_FROM_USER ) +/* protect user space index */ + prod =smp_load_acquire (&q->buf->producer_index); +else + prod = q->buf->producer_index; -/* make sure all changes to queue -* complete before getting producer index -*/ - index =smp_load_acquire (&q->buf->producer_index); - index &= q->index_mask; + prod &= q->index_mask; -return index; +return prod; }static inline unsigned int consumer_index (struct rxe_queue *q) { -u32 index; +u32 cons; -/* make sure all changes to queue -* complete before getting consumer index -*/ - index =smp_load_acquire (&q->buf->consumer_index); - index &= q->index_mask; +if (q->type ==QUEUE_TYPE_TO_USER ) +/* protect user space index */ + cons =smp_load_acquire (&q->buf->consumer_index); +else + cons = q->buf->consumer_index; -return index; + cons &= q->index_mask; + +return cons; } -static inline void *addr_from_index (struct rxe_queue *q,unsigned int index) +static inline void *addr_from_index (struct rxe_queue *q, +unsigned int index) {return q->buf->data + ((index & q->index_mask) << q->buf->log2_elem_size); }static inline unsigned int index_from_addr (const struct rxe_queue *q, -const void *addr) +const void *addr) {return (((u8 *)addr - q->buf->data) >> q->log2_elem_size) - & q->index_mask; -} - -static inline unsigned int queue_count (const struct rxe_queue *q) -{ -return (q->buf->producer_index - q->buf->consumer_index) - & q->index_mask; + & q->index_mask; }static inline void *queue_head (struct rxe_queue *q)
Signature · 1 line
2.30.2
Zhu Yanjun
maintainerRe: [PATCH for-next v2 2/2] RDMA/rxe: Protect user space index loads/stores
Quoted 91 lines · Bob Pearson
Modify the queue APIs to protect all user space index loads with
smp_load_acquire()and all user space index stores withsmp_store_release(). Base this on the types of the queues which can be one of ..KERNEL, ..FROM_USER, ..TO_USER. Kernel space indices are protected by locks which also provide memory barriers.Signed-off-by Bob Pearson <[email protected]>---
v2:
In v2 use queue type to selectively protect user space indices.--- drivers/infiniband/sw/rxe/rxe_queue.h | 168 ++++++++++++++++++-------- 1 file changed, 117 insertions(+), 51 deletions(-)drivers/infiniband/sw/rxe/rxe_queue.h +40
-14@@ -66,12 +66,22 @@ static inline int queue_empty(struct rxe_queue *q)u32 prod;u32 cons; -/* make sure all changes to queue complete before -* testing queue empty -*/ - prod =smp_load_acquire (&q->buf->producer_index); -/* same */ - cons =smp_load_acquire (&q->buf->consumer_index); +switch (q->type) { +case QUEUE_TYPE_FROM_USER : +/* protect user space index */ + prod =smp_load_acquire (&q->buf->producer_index); + cons = q->buf->consumer_index; +break ; +case QUEUE_TYPE_TO_USER : + prod = q->buf->producer_index; +/* protect user space index */ + cons =smp_load_acquire (&q->buf->consumer_index); +break ; +case QUEUE_TYPE_KERNEL : + prod = q->buf->producer_index; + cons = q->buf->consumer_index; +break ; + }return ((prod - cons) & q->index_mask) ==0 ; }@@ -81,95 +91,151 @@ static inline int queue_full(struct rxe_queue *q)u32 prod;u32 cons; -/* make sure all changes to queue complete before -* testing queue full -*/ - prod =smp_load_acquire (&q->buf->producer_index); -/* same */ - cons =smp_load_acquire (&q->buf->consumer_index); +switch (q->type) { +case QUEUE_TYPE_FROM_USER : +/* protect user space index */ + prod =smp_load_acquire (&q->buf->producer_index); + cons = q->buf->consumer_index; +break ; +case QUEUE_TYPE_TO_USER : + prod = q->buf->producer_index; +/* protect user space index */ + cons =smp_load_acquire (&q->buf->consumer_index); +break ; +case QUEUE_TYPE_KERNEL : + prod = q->buf->producer_index; + cons = q->buf->consumer_index; +break ; + }return ((prod +1 - cons) & q->index_mask) ==0 ; } -static inline void advance_producer (struct rxe_queue *q) +static inline unsigned int queue_count (const struct rxe_queue *q) {u32 prod; +u32 cons; - prod = (q->buf->producer_index +1 ) & q->index_mask; +switch (q->type) { +case QUEUE_TYPE_FROM_USER : +/* protect user space index */ + prod =smp_load_acquire (&q->buf->producer_index); + cons = q->buf->consumer_index; +break ;
drivers/infiniband/sw/rxe/rxe_queue.h static inline int queue_full(struct rxe_queue *q) · line 134
Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_queue.h +10
-0+case QUEUE_TYPE_TO_USER : + prod = q->buf->producer_index; +/* protect user space index */ + cons =smp_load_acquire (&q->buf->consumer_index); +break ; +case QUEUE_TYPE_KERNEL : + prod = q->buf->producer_index; + cons = q->buf->consumer_index; +break ; + }
The above source code appears in the functions queue_count, queue_full and queue_empty.
So is it possible to use a seperate function to implement it?
Thanks
Zhu Yanjun
Quoted 147 lines
drivers/infiniband/sw/rxe/rxe_queue.h +67
-38+ +return (prod - cons) & q->index_mask; +} + +static inline void advance_producer (struct rxe_queue *q) +{ +u32 prod; -/* make sure all changes to queue complete before -* changing producer index -*/ -smp_store_release (&q->buf->producer_index, prod); +if (q->type ==QUEUE_TYPE_FROM_USER ) { +/* protect user space index */ + prod =smp_load_acquire (&q->buf->producer_index); + prod = (prod +1 ) & q->index_mask; +/* same */ +smp_store_release (&q->buf->producer_index, prod); + }else { + prod = q->buf->producer_index; + q->buf->producer_index = (prod +1 ) & q->index_mask; + } }static inline void advance_consumer (struct rxe_queue *q) {u32 cons; - cons = (q->buf->consumer_index +1 ) & q->index_mask; - -/* make sure all changes to queue complete before -* changing consumer index -*/ -smp_store_release (&q->buf->consumer_index, cons); +if (q->type ==QUEUE_TYPE_TO_USER ) { +/* protect user space index */ + cons =smp_load_acquire (&q->buf->consumer_index); + cons = (cons +1 ) & q->index_mask; +/* same */ +smp_store_release (&q->buf->consumer_index, cons); + }else { + cons = q->buf->consumer_index; + q->buf->consumer_index = (cons +1 ) & q->index_mask; + } }static inline void *producer_addr (struct rxe_queue *q) { -return q->buf->data + ((q->buf->producer_index & q->index_mask) - << q->log2_elem_size); +u32 prod; + +if (q->type ==QUEUE_TYPE_FROM_USER ) +/* protect user space index */ + prod =smp_load_acquire (&q->buf->producer_index); +else + prod = q->buf->producer_index; + +return q->buf->data + ((prod & q->index_mask) << q->log2_elem_size); }static inline void *consumer_addr (struct rxe_queue *q) { -return q->buf->data + ((q->buf->consumer_index & q->index_mask) - << q->log2_elem_size); +u32 cons; + +if (q->type ==QUEUE_TYPE_TO_USER ) +/* protect user space index */ + cons =smp_load_acquire (&q->buf->consumer_index); +else + cons = q->buf->consumer_index; + +return q->buf->data + ((cons & q->index_mask) << q->log2_elem_size); }static inline unsigned int producer_index (struct rxe_queue *q) { -u32 index; +u32 prod; + +if (q->type ==QUEUE_TYPE_FROM_USER ) +/* protect user space index */ + prod =smp_load_acquire (&q->buf->producer_index); +else + prod = q->buf->producer_index; -/* make sure all changes to queue -* complete before getting producer index -*/ - index =smp_load_acquire (&q->buf->producer_index); - index &= q->index_mask; + prod &= q->index_mask; -return index; +return prod; }static inline unsigned int consumer_index (struct rxe_queue *q) { -u32 index; +u32 cons; -/* make sure all changes to queue -* complete before getting consumer index -*/ - index =smp_load_acquire (&q->buf->consumer_index); - index &= q->index_mask; +if (q->type ==QUEUE_TYPE_TO_USER ) +/* protect user space index */ + cons =smp_load_acquire (&q->buf->consumer_index); +else + cons = q->buf->consumer_index; -return index; + cons &= q->index_mask; + +return cons; } -static inline void *addr_from_index (struct rxe_queue *q,unsigned int index) +static inline void *addr_from_index (struct rxe_queue *q, +unsigned int index) {return q->buf->data + ((index & q->index_mask) << q->buf->log2_elem_size); }static inline unsigned int index_from_addr (const struct rxe_queue *q, -const void *addr) +const void *addr) {return (((u8 *)addr - q->buf->data) >> q->log2_elem_size) - & q->index_mask; -} - -static inline unsigned int queue_count (const struct rxe_queue *q) -{ -return (q->buf->producer_index - q->buf->consumer_index) - & q->index_mask; + & q->index_mask; }static inline void *queue_head (struct rxe_queue *q) --2.30.2
Re: [PATCH for-next v2 2/2] RDMA/rxe: Protect user space index loads/stores
Quoted 105 lines · Zhu Yanjun
Quoted 90 lines · Bob Pearson
Modify the queue APIs to protect all user space index loads with
smp_load_acquire()and all user space index stores withsmp_store_release(). Base this on the types of the queues which can be one of ..KERNEL, ..FROM_USER, ..TO_USER. Kernel space indices are protected by locks which also provide memory barriers.Signed-off-by Bob Pearson <[email protected]>---
v2:
In v2 use queue type to selectively protect user space indices.--- drivers/infiniband/sw/rxe/rxe_queue.h | 168 ++++++++++++++++++-------- 1 file changed, 117 insertions(+), 51 deletions(-)drivers/infiniband/sw/rxe/rxe_queue.h +40
-14@@ -66,12 +66,22 @@ static inline int queue_empty(struct rxe_queue *q)u32 prod;u32 cons; -/* make sure all changes to queue complete before -* testing queue empty -*/ - prod =smp_load_acquire (&q->buf->producer_index); -/* same */ - cons =smp_load_acquire (&q->buf->consumer_index); +switch (q->type) { +case QUEUE_TYPE_FROM_USER : +/* protect user space index */ + prod =smp_load_acquire (&q->buf->producer_index); + cons = q->buf->consumer_index; +break ; +case QUEUE_TYPE_TO_USER : + prod = q->buf->producer_index; +/* protect user space index */ + cons =smp_load_acquire (&q->buf->consumer_index); +break ; +case QUEUE_TYPE_KERNEL : + prod = q->buf->producer_index; + cons = q->buf->consumer_index; +break ; + }return ((prod - cons) & q->index_mask) ==0 ; }@@ -81,95 +91,151 @@ static inline int queue_full(struct rxe_queue *q)u32 prod;u32 cons; -/* make sure all changes to queue complete before -* testing queue full -*/ - prod =smp_load_acquire (&q->buf->producer_index); -/* same */ - cons =smp_load_acquire (&q->buf->consumer_index); +switch (q->type) { +case QUEUE_TYPE_FROM_USER : +/* protect user space index */ + prod =smp_load_acquire (&q->buf->producer_index); + cons = q->buf->consumer_index; +break ; +case QUEUE_TYPE_TO_USER : + prod = q->buf->producer_index; +/* protect user space index */ + cons =smp_load_acquire (&q->buf->consumer_index); +break ; +case QUEUE_TYPE_KERNEL : + prod = q->buf->producer_index; + cons = q->buf->consumer_index; +break ; + }return ((prod +1 - cons) & q->index_mask) ==0 ; } -static inline void advance_producer (struct rxe_queue *q) +static inline unsigned int queue_count (const struct rxe_queue *q) {u32 prod; +u32 cons; - prod = (q->buf->producer_index +1 ) & q->index_mask; +switch (q->type) { +case QUEUE_TYPE_FROM_USER : +/* protect user space index */ + prod =smp_load_acquire (&q->buf->producer_index); + cons = q->buf->consumer_index; +break ;drivers/infiniband/sw/rxe/rxe_queue.h static inline int queue_full(struct rxe_queue *q) · line 134
Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_queue.h +10
-0+case QUEUE_TYPE_TO_USER : + prod = q->buf->producer_index; +/* protect user space index */ + cons =smp_load_acquire (&q->buf->consumer_index); +break ; +case QUEUE_TYPE_KERNEL : + prod = q->buf->producer_index; + cons = q->buf->consumer_index; +break ; + }The above source code appears in the functions
queue_count,queue_fullandqueue_empty.
So is it possible to use a seperate function to implement it?
Quoted 2 lines
Thanks
Zhu Yanjun
Maybe, but I was trying to keep this inline. It won't save anything at runtime.
Bob
Quoted 148 lines
Quoted 148 lines
drivers/infiniband/sw/rxe/rxe_queue.h +67
-38+ +return (prod - cons) & q->index_mask; +} + +static inline void advance_producer (struct rxe_queue *q) +{ +u32 prod; -/* make sure all changes to queue complete before -* changing producer index -*/ -smp_store_release (&q->buf->producer_index, prod); +if (q->type ==QUEUE_TYPE_FROM_USER ) { +/* protect user space index */ + prod =smp_load_acquire (&q->buf->producer_index); + prod = (prod +1 ) & q->index_mask; +/* same */ +smp_store_release (&q->buf->producer_index, prod); + }else { + prod = q->buf->producer_index; + q->buf->producer_index = (prod +1 ) & q->index_mask; + } }static inline void advance_consumer (struct rxe_queue *q) {u32 cons; - cons = (q->buf->consumer_index +1 ) & q->index_mask; - -/* make sure all changes to queue complete before -* changing consumer index -*/ -smp_store_release (&q->buf->consumer_index, cons); +if (q->type ==QUEUE_TYPE_TO_USER ) { +/* protect user space index */ + cons =smp_load_acquire (&q->buf->consumer_index); + cons = (cons +1 ) & q->index_mask; +/* same */ +smp_store_release (&q->buf->consumer_index, cons); + }else { + cons = q->buf->consumer_index; + q->buf->consumer_index = (cons +1 ) & q->index_mask; + } }static inline void *producer_addr (struct rxe_queue *q) { -return q->buf->data + ((q->buf->producer_index & q->index_mask) - << q->log2_elem_size); +u32 prod; + +if (q->type ==QUEUE_TYPE_FROM_USER ) +/* protect user space index */ + prod =smp_load_acquire (&q->buf->producer_index); +else + prod = q->buf->producer_index; + +return q->buf->data + ((prod & q->index_mask) << q->log2_elem_size); }static inline void *consumer_addr (struct rxe_queue *q) { -return q->buf->data + ((q->buf->consumer_index & q->index_mask) - << q->log2_elem_size); +u32 cons; + +if (q->type ==QUEUE_TYPE_TO_USER ) +/* protect user space index */ + cons =smp_load_acquire (&q->buf->consumer_index); +else + cons = q->buf->consumer_index; + +return q->buf->data + ((cons & q->index_mask) << q->log2_elem_size); }static inline unsigned int producer_index (struct rxe_queue *q) { -u32 index; +u32 prod; + +if (q->type ==QUEUE_TYPE_FROM_USER ) +/* protect user space index */ + prod =smp_load_acquire (&q->buf->producer_index); +else + prod = q->buf->producer_index; -/* make sure all changes to queue -* complete before getting producer index -*/ - index =smp_load_acquire (&q->buf->producer_index); - index &= q->index_mask; + prod &= q->index_mask; -return index; +return prod; }static inline unsigned int consumer_index (struct rxe_queue *q) { -u32 index; +u32 cons; -/* make sure all changes to queue -* complete before getting consumer index -*/ - index =smp_load_acquire (&q->buf->consumer_index); - index &= q->index_mask; +if (q->type ==QUEUE_TYPE_TO_USER ) +/* protect user space index */ + cons =smp_load_acquire (&q->buf->consumer_index); +else + cons = q->buf->consumer_index; -return index; + cons &= q->index_mask; + +return cons; } -static inline void *addr_from_index (struct rxe_queue *q,unsigned int index) +static inline void *addr_from_index (struct rxe_queue *q, +unsigned int index) {return q->buf->data + ((index & q->index_mask) << q->buf->log2_elem_size); }static inline unsigned int index_from_addr (const struct rxe_queue *q, -const void *addr) +const void *addr) {return (((u8 *)addr - q->buf->data) >> q->log2_elem_size) - & q->index_mask; -} - -static inline unsigned int queue_count (const struct rxe_queue *q) -{ -return (q->buf->producer_index - q->buf->consumer_index) - & q->index_mask; + & q->index_mask; }static inline void *queue_head (struct rxe_queue *q) --2.30.2
Jason Gunthorpe
maintainerRe: [PATCH for-next v2 2/2] RDMA/rxe: Protect user space index loads/stores
In reply to Bob PearsonQuoted 20 lines · Bob Pearson
Modify the queue APIs to protect all user space index loads with
smp_load_acquire()and all user space index stores withsmp_store_release(). Base this on the types of the queues which can be one of ..KERNEL, ..FROM_USER, ..TO_USER. Kernel space indices are protected by locks which also provide memory barriers.Signed-off-by Bob Pearson <[email protected]>v2:
In v2 use queue type to selectively protect user space indices.drivers/infiniband/sw/rxe/rxe_queue.h | 168 ++++++++++++++++++-------- 1 file changed, 117 insertions(+), 51 deletions(-)drivers/infiniband/sw/rxe/rxe_queue.h +1
-1+++ b/drivers/infiniband/sw/rxe/rxe_queue.h@@ -66,12 +66,22 @@ static inline int queue_empty(struct rxe_queue *q)u32 prod;u32 cons; -/* make sure all changes to queue complete before
drivers/infiniband/sw/rxe/rxe_queue.h static inline int queue_empty(struct rxe_queue *q) · line 73
Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_queue.h +5
-5-* testing queue empty -*/ - prod =smp_load_acquire (&q->buf->producer_index); -/* same */ - cons =smp_load_acquire (&q->buf->consumer_index); +switch (q->type) { +case QUEUE_TYPE_FROM_USER : +/* protect user space index */ + prod =smp_load_acquire (&q->buf->producer_index); + cons = q->buf->consumer_index;
The other issue is you can't store the kernel owned consumer_index in the 'buf'
It should be stored in 'q' and only on write copied to be buf
Kernel never reads the user memory it writes to
This is why splitting it makes sense because it really needs to be reading different memory, not just using the correct load primitive
Jason