nvmet/fcloop fixes
Subsystems (3)
cover nvmet/fcloop fixes
Hi James,
here are some fixes to be applied on top of your recent patchset for nvme-fc (or even to be included with it). With these fixes my fcloop blktest runs without issues and I can even unload the fcloop module afterwards.
As usual, comments and reviews are welcome.
Hannes Reinecke (3):
nvmet/fc: Sanitize tgtport reference counting for LS requests nvme/fcloop: short-circuit LS requests if no association is present nvme/fcloop: flush workqueue before calling
nvme_fc_unregister_remoteport()
drivers/nvme/target/fc.c | 23 ++++++++++++----------- drivers/nvme/target/fcloop.c | 38 ++++++++++---------------------------- 2 files changed, 22 insertions(+), 39 deletions(-)
Signature · 7 lines
2.16.4 _______________________________________________ linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
PATCH 1/3 nvmet/fc: Sanitize tgtport reference counting for LS requests
We need to validate the targetport upon first access, otherwise there's a risk of accessing an invalid targetport.
--- drivers/nvme/target/fc.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-)
drivers/nvme/target/fc.c +12 -11
@@ -358,6 +358,7 @@ __nvmet_fc_finish_ls_req(struct nvmet_fc_ls_req_op *lsop)if (!lsop->req_queued) {spin_unlock_irqrestore (&tgtport->lock, flags); +nvmet_fc_tgtport_put (tgtport);return ; }
@@ -386,9 +387,6 @@ __nvmet_fc_send_ls_req(struct nvmet_fc_tgtport *tgtport,if (!tgtport->ops->ls_req)return -EOPNOTSUPP ; -if (!nvmet_fc_tgtport_get (tgtport)) -return -ESHUTDOWN ; - lsreq->done = done; lsop->req_queued = false;INIT_LIST_HEAD (&lsop->lsreq_list);
@@ -396,10 +394,9 @@ __nvmet_fc_send_ls_req(struct nvmet_fc_tgtport *tgtport, lsreq->rqstdma =fc_dma_map_single (tgtport->dev, lsreq->rqstaddr, lsreq->rqstlen + lsreq->rsplen,DMA_BIDIRECTIONAL ); -if (fc_dma_mapping_error (tgtport->dev, lsreq->rqstdma)) { - ret = -EFAULT ; - goto out_puttgtport; - } +if (fc_dma_mapping_error (tgtport->dev, lsreq->rqstdma)) +return -EFAULT ; + lsreq->rspdma = lsreq->rqstdma + lsreq->rqstlen;spin_lock_irqsave (&tgtport->lock, flags);
@@ -426,9 +423,6 @@ __nvmet_fc_send_ls_req(struct nvmet_fc_tgtport *tgtport,fc_dma_unmap_single (tgtport->dev, lsreq->rqstdma, (lsreq->rqstlen + lsreq->rsplen),DMA_BIDIRECTIONAL ); -out_puttgtport: -nvmet_fc_tgtport_put (tgtport); -return ret; }
@@ -482,14 +476,19 @@ nvmet_fc_xmt_disconnect_assoc(struct nvmet_fc_tgt_assoc *assoc)struct nvmefc_ls_req *lsreq;int ret; +if (!nvmet_fc_tgtport_get (tgtport)) +return ; +/* * If ls_req is NULL or no hosthandle, it's an older lldd and no * message is normal. Otherwise, send unless the hostport has * already been invalidated by the lldd. */ if (!tgtport->ops->ls_req || !assoc->hostport || - assoc->hostport->invalid) + assoc->hostport->invalid) { +nvmet_fc_tgtport_put (tgtport);return ; + } lsop = kzalloc((sizeof(*lsop) + sizeof(*discon_rqst) + sizeof(*discon_acc) +
@@ -498,6 +497,7 @@ nvmet_fc_xmt_disconnect_assoc(struct nvmet_fc_tgt_assoc *assoc)dev_info (tgtport->dev,"{%d:%d} send Disconnect Association failed: ENOMEM\n" , tgtport->fc_target_port.port_num, assoc->a_id); +nvmet_fc_tgtport_put (tgtport);return ; }
@@ -522,6 +522,7 @@ nvmet_fc_xmt_disconnect_assoc(struct nvmet_fc_tgt_assoc *assoc)"{%d:%d} XMT Disconnect Association failed: %d\n" , tgtport->fc_target_port.port_num, assoc->a_id, ret);kfree (lsop); +nvmet_fc_tgtport_put (tgtport); } }
Signature · 7 lines
2.16.4 _______________________________________________ linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
Re: [PATCH 1/3] nvmet/fc: Sanitize tgtport reference counting for LS requests
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Re: [PATCH 1/3] nvmet/fc: Sanitize tgtport reference counting for LS requests
In reply to Hannes ReineckeQuoted 84 lines · Hannes Reinecke
We need to validate the targetport upon first access, otherwise there's a risk of accessing an invalid targetport.
Signed-off-by Hannes Reinecke <[email protected]>--- drivers/nvme/target/fc.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-)drivers/nvme/target/fc.c +11
-11@@ -358,6 +358,7 @@ __nvmet_fc_finish_ls_req(struct nvmet_fc_ls_req_op *lsop)if (!lsop->req_queued) {spin_unlock_irqrestore (&tgtport->lock, flags); +nvmet_fc_tgtport_put (tgtport);return ; }@@ -386,9 +387,6 @@ __nvmet_fc_send_ls_req(struct nvmet_fc_tgtport *tgtport,if (!tgtport->ops->ls_req)return -EOPNOTSUPP ; -if (!nvmet_fc_tgtport_get (tgtport)) -return -ESHUTDOWN ; - lsreq->done = done; lsop->req_queued = false;INIT_LIST_HEAD (&lsop->lsreq_list);@@ -396,10 +394,9 @@ __nvmet_fc_send_ls_req(struct nvmet_fc_tgtport *tgtport, lsreq->rqstdma =fc_dma_map_single (tgtport->dev, lsreq->rqstaddr, lsreq->rqstlen + lsreq->rsplen,DMA_BIDIRECTIONAL ); -if (fc_dma_mapping_error (tgtport->dev, lsreq->rqstdma)) { - ret = -EFAULT ; - goto out_puttgtport; - } +if (fc_dma_mapping_error (tgtport->dev, lsreq->rqstdma)) +return -EFAULT ; + lsreq->rspdma = lsreq->rqstdma + lsreq->rqstlen;spin_lock_irqsave (&tgtport->lock, flags);@@ -426,9 +423,6 @@ __nvmet_fc_send_ls_req(struct nvmet_fc_tgtport *tgtport,fc_dma_unmap_single (tgtport->dev, lsreq->rqstdma, (lsreq->rqstlen + lsreq->rsplen),DMA_BIDIRECTIONAL ); -out_puttgtport: -nvmet_fc_tgtport_put (tgtport); -return ret; }@@ -482,14 +476,19 @@ nvmet_fc_xmt_disconnect_assoc(struct nvmet_fc_tgt_assoc *assoc)struct nvmefc_ls_req *lsreq;int ret; +if (!nvmet_fc_tgtport_get (tgtport)) +return ; +/* * If ls_req is NULL or no hosthandle, it's an older lldd and no * message is normal. Otherwise, send unless the hostport has * already been invalidated by the lldd. */ if (!tgtport->ops->ls_req || !assoc->hostport || - assoc->hostport->invalid) + assoc->hostport->invalid) { +nvmet_fc_tgtport_put (tgtport);return ; + } lsop = kzalloc((sizeof(*lsop) + sizeof(*discon_rqst) + sizeof(*discon_acc) +@@ -498,6 +497,7 @@ nvmet_fc_xmt_disconnect_assoc(struct nvmet_fc_tgt_assoc *assoc)dev_info (tgtport->dev,"{%d:%d} send Disconnect Association failed: ENOMEM\n" , tgtport->fc_target_port.port_num, assoc->a_id); +nvmet_fc_tgtport_put (tgtport);return ; }
drivers/nvme/target/fc.c nvmet_fc_xmt_disconnect_assoc(struct nvmet_fc_tgt_assoc *assoc) · line 528
Quoted 7 lines
drivers/nvme/target/fc.c +1
-0@@ -522,6 +522,7 @@ nvmet_fc_xmt_disconnect_assoc(struct nvmet_fc_tgt_assoc *assoc)"{%d:%d} XMT Disconnect Association failed: %d\n" , tgtport->fc_target_port.port_num, assoc->a_id, ret);kfree (lsop); +nvmet_fc_tgtport_put (tgtport); } }
I guess I don't see anything wrong with this, but it seems messier to handle the ref counting. I prefer it to be on the send/completion localized to the common op processing. Also creates a different implementation than what was done on the host side, even though the rest is very similar.
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Christoph Hellwig
maintainerRe: [PATCH 1/3] nvmet/fc: Sanitize tgtport reference counting for LS requests
In reply to Hannes ReineckeQuoted 30 lines
drivers/nvme/target/fc.c +7
-1@@ -482,14 +476,19 @@ nvmet_fc_xmt_disconnect_assoc(struct nvmet_fc_tgt_assoc *assoc)struct nvmefc_ls_req *lsreq;int ret; +if (!nvmet_fc_tgtport_get (tgtport)) +return ; +/* * If ls_req is NULL or no hosthandle, it's an older lldd and no * message is normal. Otherwise, send unless the hostport has * already been invalidated by the lldd. */ if (!tgtport->ops->ls_req || !assoc->hostport || - assoc->hostport->invalid) + assoc->hostport->invalid) { +nvmet_fc_tgtport_put (tgtport);return ; + } lsop = kzalloc((sizeof(*lsop) + sizeof(*discon_rqst) + sizeof(*discon_acc) +@@ -498,6 +497,7 @@ nvmet_fc_xmt_disconnect_assoc(struct nvmet_fc_tgt_assoc *assoc)dev_info (tgtport->dev,"{%d:%d} send Disconnect Association failed: ENOMEM\n" , tgtport->fc_target_port.port_num, assoc->a_id); +nvmet_fc_tgtport_put (tgtport);return ; }
drivers/nvme/target/fc.c nvmet_fc_xmt_disconnect_assoc(struct nvmet_fc_tgt_assoc *assoc) · line 525
Quoted 4 lines
drivers/nvme/target/fc.c +1
-0@@ -522,6 +522,7 @@ nvmet_fc_xmt_disconnect_assoc(struct nvmet_fc_tgt_assoc *assoc)"{%d:%d} XMT Disconnect Association failed: %d\n" , tgtport->fc_target_port.port_num, assoc->a_id, ret);kfree (lsop); +nvmet_fc_tgtport_put (tgtport);
Can you consolidate all the calls in this function using goto unwinding?
Otherwise looks good.
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
PATCH 2/3 nvme/fcloop: short-circuit LS requests if no association is present
In reply to Hannes ReineckeIf no association or targetport is present shifting LS processing to a workqueue is a bad idea as the targetport itself is about to be removed, and we would deadlock with targetport deletion. So instead return an error directly and treat it as a send error, avoiding the deadlock.
--- drivers/nvme/target/fcloop.c | 36 +++++++++--------------------------- 1 file changed, 9 insertions(+), 27 deletions(-)
drivers/nvme/target/fcloop.c +9 -27
@@ -332,26 +332,17 @@ fcloop_h2t_ls_req(struct nvme_fc_local_port *localport, {struct fcloop_lsreq *tls_req = lsreq->private;struct fcloop_rport *rport = remoteport->private; -int ret =0 ; + +if (!rport->targetport) +return -ECONNREFUSED ; tls_req->lsreq = lsreq;INIT_LIST_HEAD (&tls_req->ls_list); -if (!rport->targetport) { - tls_req->status = -ECONNREFUSED ; -spin_lock (&rport->lock); -list_add_tail (&rport->ls_list, &tls_req->ls_list); -spin_unlock (&rport->lock); -schedule_work (&rport->ls_work); -return ret; - } - tls_req->status =0 ; - ret =nvmet_fc_rcv_ls_req (rport->targetport, rport, +return nvmet_fc_rcv_ls_req (rport->targetport, rport, &tls_req->ls_rsp, lsreq->rqstaddr, lsreq->rqstlen); - -return ret; } static int
@@ -415,7 +406,9 @@ fcloop_t2h_ls_req(struct nvmet_fc_target_port *targetport, void *hosthandle, {struct fcloop_lsreq *tls_req = lsreq->private;struct fcloop_tport *tport = targetport->private; -int ret =0 ; + +if (!tport->remoteport) +return -ECONNREFUSED ;/* * hosthandle should be the dst.rport value.
@@ -425,20 +418,9 @@ fcloop_t2h_ls_req(struct nvmet_fc_target_port *targetport, void *hosthandle, tls_req->lsreq = lsreq;INIT_LIST_HEAD (&tls_req->ls_list); -if (!tport->remoteport) { - tls_req->status = -ECONNREFUSED ; -spin_lock (&tport->lock); -list_add_tail (&tport->ls_list, &tls_req->ls_list); -spin_unlock (&tport->lock); -schedule_work (&tport->ls_work); -return ret; - } - tls_req->status =0 ; - ret =nvme_fc_rcv_ls_req (tport->remoteport, &tls_req->ls_rsp, - lsreq->rqstaddr, lsreq->rqstlen); - -return ret; +return nvme_fc_rcv_ls_req (tport->remoteport, &tls_req->ls_rsp, + lsreq->rqstaddr, lsreq->rqstlen); } static int
Signature · 7 lines
2.16.4 _______________________________________________ linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
Re: [PATCH 2/3] nvme/fcloop: short-circuit LS requests if no association is present
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Re: [PATCH 2/3] nvme/fcloop: short-circuit LS requests if no association is present
In reply to Hannes ReineckeQuoted 5 lines · Hannes Reinecke
If no association or targetport is present shifting LS processing to a workqueue is a bad idea as the targetport itself is about to be removed, and we would deadlock with targetport deletion. So instead return an error directly and treat it as a send error, avoiding the deadlock.
So I don't quite agree on the targetport statement, especially relative to both the h2t and t2h paths.
Returning error certainly works, but it avoids the asynchronous completion path which is far more common. It only tests the immediate-return-error code case. When throwing it to a work queue, it'll be a work queue specific to the port initiating the request - so it shouldn't be touching the other side which is non-existent.
so what you have fine, but i'm not sure it's what should be here.
-- james
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Re: [PATCH 2/3] nvme/fcloop: short-circuit LS requests if no association is present
Quoted 6 lines · James Smart
Quoted 5 lines · Hannes Reinecke
If no association or targetport is present shifting LS processing to a workqueue is a bad idea as the targetport itself is about to be removed, and we would deadlock with targetport deletion. So instead return an error directly and treat it as a send error, avoiding the deadlock.
Quoted 6 lines
So I don't quite agree on the targetport statement, especially relative to both the h2t and t2h paths.
Returning error certainly works, but it avoids the asynchronous completion path which is far more common. It only tests the immediate-return-error code case. When throwing it to a work queue, it'll be a work queue specific to the port initiating the request - so it shouldn't be touching the other side which is non-existent.
so what you have fine, but i'm not sure it's what should be here.
Ah. you spotted it :-)
You are correct, it doesn't fix the underlying problem. But it does allow targetport deletion to finish, so there is _some_ value in it.
But indeed, the real problem is the flush_work() statement infcloop_targetport_delete(). Thing is, fcloop_targetport_delete() is called from the very same workqueue function when processing ls requests, and those need to flushed when tearing down the targetport:
[ 1488.211329] Workqueue: events fcloop_tport_lsrqst_work [ nvme_fcloop] [ 1488.211331] Call Trace: [ 1488.369706] schedule+0x40/0xb0 [ 1488.369708] schedule_timeout+0x1dd/0x300 [ 1488.469700] wait_for_completion+0xba/0x140 [ 1488.469705] __flush_work+0x177/0x1b0 [ 1488.469708] ? worker_detach_from_pool+0xa0/0xa0 [ 1488.610280] fcloop_targetport_delete+0x13/0x20 [nvme_fcloop] [ 1488.610284] nvmet_fc_tgtport_put+0x150/0x190 [nvmet_fc] [ 1488.706952] nvmet_fc_disconnect_assoc_done+0x9a/0xe0 [nvmet_fc] [ 1488.706954] fcloop_tport_lsrqst_work+0x7a/0xa0 [nvme_fcloop] [ 1488.706957] process_one_work+0x208/0x400 [ 1488.706959] worker_thread+0x2d/0x3e0
So that needs to be move to somewhere else, but I haven't followed the flow of control that closely that I've been able to figure out _where_ it should go to.
Cheers,
Hannes
Signature · 9 lines
Dr. Hannes Reinecke Teamlead Storage & Networking [email protected] +49 911 74053 688 SUSE Software Solutions GmbH, Maxfeldstr. 5, 90409 Nürnberg HRB 36809 (AG Nürnberg), Geschäftsführer: Felix Imendörffer _______________________________________________ linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
PATCH 3/3 nvme/fcloop: flush workqueue before calling nvme_fc_unregister_remoteport()
In reply to Hannes Reineckenvme_fc_unregister_remoteport() will be sending LS requests, which then would end up on a workqueue for processing. This will deadlock with fcloop_remoteport_delete() which would try to flush the very same queue.
--- drivers/nvme/target/fcloop.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/nvme/target/fcloop.c +1 -1
@@ -943,7 +943,6 @@ fcloop_remoteport_delete(struct nvme_fc_remote_port *remoteport) {struct fcloop_rport *rport = remoteport->private; -flush_work (&rport->ls_work);fcloop_nport_put (rport->nport); }
@@ -1278,6 +1277,7 @@ __remoteport_unreg(struct fcloop_nport *nport, struct fcloop_rport *rport)if (!rport)return -EALREADY ; +flush_work (&rport->ls_work);return nvme_fc_unregister_remoteport (rport->remoteport); }
Signature · 7 lines
2.16.4 _______________________________________________ linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
Re: [PATCH 3/3] nvme/fcloop: flush workqueue before calling nvme_fc_unregister_remoteport()
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Re: [PATCH 3/3] nvme/fcloop: flush workqueue before calling nvme_fc_unregister_remoteport()
In reply to Hannes ReineckeQuoted 22 lines · Hannes Reinecke
nvme_fc_unregister_remoteport()will be sending LS requests, which then would end up on a workqueue for processing. This will deadlock withfcloop_remoteport_delete()which would try to flush the very same queue.Signed-off-by Hannes Reinecke <[email protected]>--- drivers/nvme/target/fcloop.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)drivers/nvme/target/fcloop.c +0
-1@@ -943,7 +943,6 @@ fcloop_remoteport_delete(struct nvme_fc_remote_port *remoteport) {struct fcloop_rport *rport = remoteport->private; -flush_work (&rport->ls_work);fcloop_nport_put (rport->nport); }
drivers/nvme/target/fcloop.c __remoteport_unreg(struct fcloop_nport *nport, struct fcloop_rport *rport) · line 1283
Quoted 7 lines
drivers/nvme/target/fcloop.c +1
-0@@ -1278,6 +1277,7 @@ __remoteport_unreg(struct fcloop_nport *nport, struct fcloop_rport *rport)if (!rport)return -EALREADY ; +flush_work (&rport->ls_work);return nvme_fc_unregister_remoteport (rport->remoteport); }
And this is what we really needed, although there's probably a like thing that should be done on the targetport as well.
-- james
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Re: [PATCH 3/3] nvme/fcloop: flush workqueue before calling nvme_fc_unregister_remoteport()
Quoted 24 lines · James Smart
Quoted 21 lines · Hannes Reinecke
nvme_fc_unregister_remoteport()will be sending LS requests, which then would end up on a workqueue for processing. This will deadlock withfcloop_remoteport_delete()which would try to flush the very same queue.Signed-off-by Hannes Reinecke <[email protected]>---
drivers/nvme/target/fcloop.c| 2 +-1 file changed, 1 insertion(+), 1 deletion(-)drivers/nvme/target/fcloop.c +0
-1@@ -943,7 +943,6 @@ fcloop_remoteport_delete(struct nvme_fc_remote_port *remoteport) { struct fcloop_rport *rport = remoteport->private; - flush_work(&rport->ls_work); fcloop_nport_put(rport->nport); }@@ -1278,6 +1277,7 @@ __remoteport_unreg(struct fcloop_nport *nport,
Quoted 9 lines
drivers/nvme/target/fcloop.c __remoteport_unreg(struct fcloop_nport *nport, struct fcloop_rport *rport) · line 1282
Quoted 6 lines
drivers/nvme/target/fcloop.c +1
-0struct fcloop_rport *rport) if (!rport) return -EALREADY; + flush_work(&rport->ls_work); return nvme_fc_unregister_remoteport(rport->remoteport); }And this is what we really needed, although there's probably a like thing that should be done on the targetport as well.
As mentioned in the previous mail: I tried, but that crashed even more horribly. Might've been to stupid to do it correctly, though.
Cheers,
Hannes
Signature · 9 lines
Dr. Hannes Reinecke Teamlead Storage & Networking [email protected] +49 911 74053 688 SUSE Software Solutions GmbH, Maxfeldstr. 5, 90409 Nürnberg HRB 36809 (AG Nürnberg), Geschäftsführer: Felix Imendörffer _______________________________________________ linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme
Christoph Hellwig
maintainerHannes,
are you going to respin this series?
_______________________________________________
linux-nvme mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-nvme
Quoted 4 lines · Christoph Hellwig
Hannes,
are you going to respin this series?
Yes, will be doing so shortly.
Cheers,
Hannes
Signature · 9 lines
Dr. Hannes Reinecke Teamlead Storage & Networking [email protected] +49 911 74053 688 SUSE Software Solutions GmbH, Maxfeldstr. 5, 90409 Nürnberg HRB 36809 (AG Nürnberg), Geschäftsführer: Felix Imendörffer _______________________________________________ linux-nvme mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-nvme