RDMA/siw: Fix handling of zero-sized Read and Receive Queues.
Subsystems (3)
PATCH v2 1/1 RDMA/siw: Fix handling of zero-sized Read and Receive Queues.
During connection setup, the application may choose to zero-size inbound and outbound READ queues, as well as the Receive queue. This patch fixes handling of zero-sized queues, but not prevents it.
v2 changes:
- Fix uninitialized variable introduced in siw_qp_rx.c, as
- Add initial error report as
Kamal Heib says in an initial error report:
When running the blktests over siw the following shift-out-of-bounds is reported, this is happening because the passed IRD or ORD from the ulp could be zero which will lead to unexpected behavior when calling roundup_pow_of_two(), fix that by blocking zero values of ORD or IRD.
UBSAN: shift-out-of-bounds in ./include/linux/log2.h:57:13 shift exponent 64 is too large for 64-bit type 'long unsigned int' CPU: 20 PID: 3957 Comm: kworker/u64:13 Tainted: G S 5.10.0-rc6 #2 Hardware name: Dell Inc. PowerEdge R630/02C2CP, BIOS 2.1.5 04/11/2016 Workqueue: iw_cm_wq cm_work_handler [iw_cm] Call Trace: dump_stack+0x99/0xcb ubsan_epilogue+0x5/0x40 __ubsan_handle_shift_out_of_bounds.cold.11+0xb4/0xf3 ? down_write+0x183/0x3d0 siw_qp_modify.cold.8+0x2d/0x32 [siw] ? __local_bh_enable_ip+0xa5/0xf0 siw_accept+0x906/0x1b60 [siw] ? xa_load+0x147/0x1f0 ? siw_connect+0x17a0/0x17a0 [siw] ? lock_downgrade+0x700/0x700 ? siw_get_base_qp+0x1c2/0x340 [siw] ? _raw_spin_unlock_irqrestore+0x39/0x40 iw_cm_accept+0x1f4/0x430 [iw_cm] rdma_accept+0x3fa/0xb10 [rdma_cm] ? check_flush_dependency+0x410/0x410 ? cma_rep_recv+0x570/0x570 [rdma_cm] nvmet_rdma_queue_connect+0x1a62/0x2680 [nvmet_rdma] ? nvmet_rdma_alloc_cmds+0xce0/0xce0 [nvmet_rdma] ? lock_release+0x56e/0xcc0 ? lock_downgrade+0x700/0x700 ? lock_downgrade+0x700/0x700 ? __xa_alloc_cyclic+0xef/0x350 ? __xa_alloc+0x2d0/0x2d0 ? rdma_restrack_add+0xbe/0x2c0 [ib_core] ? __ww_mutex_die+0x190/0x190 cma_cm_event_handler+0xf2/0x500 [rdma_cm] iw_conn_req_handler+0x910/0xcb0 [rdma_cm] ? _raw_spin_unlock_irqrestore+0x39/0x40 ? trace_hardirqs_on+0x1c/0x150 ? cma_ib_handler+0x8a0/0x8a0 [rdma_cm] ? __kasan_kmalloc.constprop.7+0xc1/0xd0 cm_work_handler+0x121c/0x17a0 [iw_cm] ? iw_cm_reject+0x190/0x190 [iw_cm] ? trace_hardirqs_on+0x1c/0x150 process_one_work+0x8fb/0x16c0 ? pwq_dec_nr_in_flight+0x320/0x320 worker_thread+0x87/0xb40 ? __kthread_parkme+0xd1/0x1a0 ? process_one_work+0x16c0/0x16c0 kthread+0x35f/0x430 ? kthread_mod_delayed_work+0x180/0x180 ret_from_fork+0x22/0x30
--- drivers/infiniband/sw/siw/siw.h | 2 +- drivers/infiniband/sw/siw/siw_qp.c | 54 ++++++++++++++++----------- drivers/infiniband/sw/siw/siw_qp_rx.c | 26 +++++++++---- drivers/infiniband/sw/siw/siw_qp_tx.c | 4 +- drivers/infiniband/sw/siw/siw_verbs.c | 18 +++++++-- 5 files changed, 68 insertions(+), 36 deletions(-)
drivers/infiniband/sw/siw/siw.h +1 -1
@@ -654,7 +654,7 @@ static inline struct siw_sqe *orq_get_free(struct siw_qp *qp) {struct siw_sqe *orq_e =orq_get_tail (qp); -if (orq_e &&READ_ONCE (orq_e->flags) ==0 ) +if (READ_ONCE (orq_e->flags) ==0 )return orq_e;return NULL ;
drivers/infiniband/sw/siw/siw_qp.c +33 -21
97 lines not on this page · raw message
drivers/infiniband/sw/siw/siw_qp_rx.c +18 -8
@@ -678,6 +678,10 @@ static int siw_init_rresp(struct siw_qp *qp, struct siw_rx_stream *srx) DDP_ECODE_UT_INVALID_MSN_RANGE, 0);return -EPROTO ; } +if (unlikely (!qp->attrs.irq_size)) { + run_sq =0 ; + goto error_irq; + }spin_lock_irqsave (&qp->sq_lock, flags);if (tx_work->wr_status ==SIW_WR_IDLE ) {
@@ -712,8 +716,9 @@ static int siw_init_rresp(struct siw_qp *qp, struct siw_rx_stream *srx)/* RRESP now valid as current TX wqe or placed into IRQ */ smp_store_mb (resp->flags,SIW_WQE_VALID ); }else { -pr_warn ("siw: [QP %u]: irq %d exceeded %d\n" ,qp_id (qp), - qp->irq_put % qp->attrs.irq_size, qp->attrs.irq_size); +error_irq: +pr_warn ("siw: [QP %u]: IRQ exceeded or null, size %d\n" , +qp_id (qp), qp->attrs.irq_size); siw_init_terminate(qp, TERM_ERROR_LAYER_RDMAP, RDMAP_ETYPE_REMOTE_OPERATION,
@@ -740,6 +745,9 @@ static int siw_orqe_start_rx(struct siw_qp *qp)struct siw_sqe *orqe;struct siw_wqe *wqe =NULL ; +if (unlikely (!qp->attrs.orq_size)) +return -EPROTO ; +/* make sure ORQ indices are current */ smp_mb ();
@@ -796,8 +804,8 @@ int siw_proc_rresp(struct siw_qp *qp)*/ rv =siw_orqe_start_rx (qp);if (rv) { -pr_warn ("siw: [QP %u]: ORQ empty at idx %d\n" , -qp_id (qp), qp->orq_get % qp->attrs.orq_size); +pr_warn ("siw: [QP %u]: ORQ empty, size %d\n" , +qp_id (qp), qp->attrs.orq_size); goto error_term; } rv =siw_rresp_check_ntoh (srx, frx);
@@ -1290,11 +1298,13 @@ static int siw_rdmap_complete(struct siw_qp *qp, int error) wc_status);siw_wqe_put_mem (wqe,SIW_OP_READ ); -if (!error) +if (!error) { rv =siw_check_tx_fence (qp); -else -/* Disable current ORQ eleement */ -WRITE_ONCE (orq_get_current (qp)->flags,0 ); + }else { +/* Disable current ORQ element */ +if (qp->attrs.orq_size) +WRITE_ONCE (orq_get_current (qp)->flags,0 ); + }break ;case RDMAP_RDMA_READ_REQ :
drivers/infiniband/sw/siw/siw_qp_tx.c +2 -2
@@ -1107,8 +1107,8 @@ int siw_qp_sq_process(struct siw_qp *qp)/* * RREQ may have already been completed by inbound RRESP! */ -if (tx_type ==SIW_OP_READ || - tx_type ==SIW_OP_READ_LOCAL_INV ) { +if ((tx_type ==SIW_OP_READ || + tx_type ==SIW_OP_READ_LOCAL_INV ) && qp->attrs.orq_size) {/* Cleanup pending entry in ORQ */ qp->orq_put--; qp->orq[qp->orq_put % qp->attrs.orq_size].flags =0 ;
drivers/infiniband/sw/siw/siw_verbs.c +14 -4
@@ -362,13 +362,23 @@ struct ib_qp *siw_create_qp(struct ib_pd *pd,if (rv) goto err_out; + num_sqe = attrs->cap.max_send_wr; + num_rqe = attrs->cap.max_recv_wr; +/* All queue indices are derived from modulo operations * on a free running 'get' (consumer) and 'put' (producer) * unsigned counter. Having queue sizes at power of two * avoids handling counter wrap around. */ - num_sqe =roundup_pow_of_two (attrs->cap.max_send_wr); - num_rqe =roundup_pow_of_two (attrs->cap.max_recv_wr); +if (num_sqe) + num_sqe =roundup_pow_of_two (num_sqe); +else { +/* Zero sized SQ is not supported */ + rv = -EINVAL ; + goto err_out; + } +if (num_rqe) + num_rqe =roundup_pow_of_two (num_rqe);if (udata) qp->sendq =vmalloc_user (num_sqe *sizeof (struct siw_sqe ));
@@ -960,9 +970,9 @@ int siw_post_receive(struct ib_qp *base_qp, const struct ib_recv_wr *wr,unsigned long flags;int rv =0 ; -if (qp->srq) { +if (qp->srq || qp->attrs.rq_size ==0 ) { *bad_wr = wr; -return -EOPNOTSUPP ;/* what else from errno.h? */ +return -EINVAL ; }if (!rdma_is_kernel_res (&qp->base_qp.res)) {siw_dbg_qp (qp,"no kernel post_recv for user mapped rq\n" );
Signature · 7 lines
2.17.2 _______________________________________________ Linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
Quoted 10 lines · Bernard Metzler
During connection setup, the application may choose to zero-size inbound and outbound READ queues, as well as the Receive queue. This patch fixes handling of zero-sized queues, but not prevents it.
v2 changes:
- Fix uninitialized variable introduced insiw_qp_rx.c, asReported-by kernel test robot <[email protected]>- Add initial error report as
Reported-by Kamal Heib <[email protected]>
Changelog shouldn't be in the commit message and needs to appear after "---", because only this version will be applied to the kernel.
Quoted 99 lines
Kamal Heib says in an initial error report:
When running the blktests over siw the following shift-out-of-bounds is reported, this is happening because the passed IRD or ORD from the ulp could be zero which will lead to unexpected behavior when callingroundup_pow_of_two(), fix that by blocking zero values of ORD or IRD.UBSAN: shift-out-of-bounds in ./include/linux/log2.h:57:13 shift exponent 64 is too large for 64-bit type 'long unsigned int' CPU: 20 PID: 3957 Comm: kworker/u64:13 Tainted: G S 5.10.0-rc6 #2 Hardware name: Dell Inc. PowerEdge R630/02C2CP, BIOS 2.1.5 04/11/2016 Workqueue: iw_cm_wq cm_work_handler [iw_cm] Call Trace: dump_stack+0x99/0xcb ubsan_epilogue+0x5/0x40 __ubsan_handle_shift_out_of_bounds.cold.11+0xb4/0xf3 ? down_write+0x183/0x3d0 siw_qp_modify.cold.8+0x2d/0x32 [siw] ? __local_bh_enable_ip+0xa5/0xf0 siw_accept+0x906/0x1b60 [siw] ? xa_load+0x147/0x1f0 ? siw_connect+0x17a0/0x17a0 [siw] ? lock_downgrade+0x700/0x700 ? siw_get_base_qp+0x1c2/0x340 [siw] ? _raw_spin_unlock_irqrestore+0x39/0x40 iw_cm_accept+0x1f4/0x430 [iw_cm] rdma_accept+0x3fa/0xb10 [rdma_cm] ? check_flush_dependency+0x410/0x410 ? cma_rep_recv+0x570/0x570 [rdma_cm] nvmet_rdma_queue_connect+0x1a62/0x2680 [nvmet_rdma] ? nvmet_rdma_alloc_cmds+0xce0/0xce0 [nvmet_rdma] ? lock_release+0x56e/0xcc0 ? lock_downgrade+0x700/0x700 ? lock_downgrade+0x700/0x700 ? __xa_alloc_cyclic+0xef/0x350 ? __xa_alloc+0x2d0/0x2d0 ? rdma_restrack_add+0xbe/0x2c0 [ib_core] ? __ww_mutex_die+0x190/0x190 cma_cm_event_handler+0xf2/0x500 [rdma_cm] iw_conn_req_handler+0x910/0xcb0 [rdma_cm] ? _raw_spin_unlock_irqrestore+0x39/0x40 ? trace_hardirqs_on+0x1c/0x150 ? cma_ib_handler+0x8a0/0x8a0 [rdma_cm] ? __kasan_kmalloc.constprop.7+0xc1/0xd0 cm_work_handler+0x121c/0x17a0 [iw_cm] ? iw_cm_reject+0x190/0x190 [iw_cm] ? trace_hardirqs_on+0x1c/0x150 process_one_work+0x8fb/0x16c0 ? pwq_dec_nr_in_flight+0x320/0x320 worker_thread+0x87/0xb40 ? __kthread_parkme+0xd1/0x1a0 ? process_one_work+0x16c0/0x16c0 kthread+0x35f/0x430 ? kthread_mod_delayed_work+0x180/0x180 ret_from_fork+0x22/0x30Fixes a531975279f3 ("rdma/siw: main include file")Fixes f29dd55b0236 ("rdma/siw: queue pair methods")Fixes 8b6a361b8c48 ("rdma/siw: receive path")Fixes b9be6f18cf9e ("rdma/siw: transmit path")Fixes 303ae1cdfdf7 ("rdma/siw: application interface")Reported-by Kamal Heib <[email protected]>Reported-by Yi Zhang <[email protected]>Signed-off-by Bernard Metzler <[email protected]>--- drivers/infiniband/sw/siw/siw.h | 2 +- drivers/infiniband/sw/siw/siw_qp.c | 54 ++++++++++++++++----------- drivers/infiniband/sw/siw/siw_qp_rx.c | 26 +++++++++---- drivers/infiniband/sw/siw/siw_qp_tx.c | 4 +- drivers/infiniband/sw/siw/siw_verbs.c | 18 +++++++-- 5 files changed, 68 insertions(+), 36 deletions(-)drivers/infiniband/sw/siw/siw.h +1
-1@@ -654,7 +654,7 @@ static inline struct siw_sqe *orq_get_free(struct siw_qp *qp) {struct siw_sqe *orq_e =orq_get_tail (qp); -if (orq_e &&READ_ONCE (orq_e->flags) ==0 ) +if (READ_ONCE (orq_e->flags) ==0 )return orq_e;return NULL ;drivers/infiniband/sw/siw/siw_qp.c +0
-6@@ -199,26 +199,28 @@ void siw_qp_llp_write_space(struct sock *sk)static int siw_qp_readq_init (struct siw_qp *qp,int irq_size,int orq_size) { - irq_size =roundup_pow_of_two (irq_size); - orq_size =roundup_pow_of_two (orq_size); - - qp->attrs.irq_size = irq_size; - qp->attrs.orq_size = orq_size; -
drivers/infiniband/sw/siw/siw_qp.c void siw_qp_llp_write_space(struct sock *sk) · line 206
Quoted 10 lines
drivers/infiniband/sw/siw/siw_qp.c +5
-5- qp->irq =vzalloc (irq_size *sizeof (struct siw_sqe )); -if (!qp->irq) { -siw_dbg_qp (qp,"irq malloc for %d failed\n" , irq_size); - qp->attrs.irq_size =0 ; -return -ENOMEM ; +if (irq_size) { + irq_size =roundup_pow_of_two (irq_size); + qp->irq =vzalloc (irq_size *sizeof (struct siw_sqe )); +if (!qp->irq) { +siw_dbg_qp (qp,"irq malloc for %d failed\n" , irq_size);
Please don't copy the prints.
Thanks
_______________________________________________
Linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme