Fix use-after-free errors in lpfc driver when using NVMe
Subsystems (3)
cover Fix use-after-free errors in lpfc driver when using NVMe
These two patches fix use-after-free errors in the shutdown path of the lpfc driver with both the Initiator and Target mode usage of FC ports.
The problem is very apparent with slub_debug enabled, as the memory poisoning prevents the wait_for_completion_timeout() from returning after the object has been freed.
Ewan D. Milne (2):
lpfc: nvme: avoid hang / use-after-free when destroying localport
lpfc: nvmet: avoid hang / use-after-free when destroying targetport
drivers/scsi/lpfc/lpfc_nvme.c | 16 +++++++++------- drivers/scsi/lpfc/lpfc_nvme.h | 2 +- drivers/scsi/lpfc/lpfc_nvmet.c | 8 +++++--- drivers/scsi/lpfc/lpfc_nvmet.h | 2 +- 4 files changed, 16 insertions(+), 12 deletions(-)
Signature · 1 line
2.1.0
PATCH 1/2 lpfc: nvme: avoid hang / use-after-free when destroying localport
We cannot wait on a completion object in the lpfc_nvme_lport structure in the _destroy_localport() code path because the NVMe/fc transport will free that structure immediately after the .localport_delete() callback. This results in a use-after-free, and a hang if slub_debug=FZPU is enabled.
Fix this by putting the completion on the stack.
--- drivers/scsi/lpfc/lpfc_nvme.c | 16 +++++++++------- drivers/scsi/lpfc/lpfc_nvme.h | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-)
drivers/scsi/lpfc/lpfc_nvme.c +9 -7
@@ -297,7 +297,8 @@ lpfc_nvme_localport_delete(struct nvme_fc_local_port *localport) lport);/* release any threads waiting for the unreg to complete */ -complete (&lport->lport_unreg_done); +if (lport->vport->localport) +complete (lport->lport_unreg_cmp); }/* lpfc_nvme_remoteport_delete
@@ -2545,7 +2546,8 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport)*/ void lpfc_nvme_lport_unreg_wait (struct lpfc_vport *vport, -struct lpfc_nvme_lport *lport) +struct lpfc_nvme_lport *lport, +struct completion *lport_unreg_cmp) {#if (IS_ENABLED (CONFIG_NVME_FC ))u32 wait_tmo;
@@ -2557,8 +2559,7 @@ lpfc_nvme_lport_unreg_wait(struct lpfc_vport *vport,*/ wait_tmo =msecs_to_jiffies (LPFC_NVME_WAIT_TMO *1000 );while (true) { - ret =wait_for_completion_timeout (&lport->lport_unreg_done, - wait_tmo); + ret =wait_for_completion_timeout (lport_unreg_cmp, wait_tmo);if (unlikely (!ret)) {lpfc_printf_vlog (vport,KERN_ERR ,LOG_NVME_IOERR , "6176 Lport %p Localport %p wait "
@@ -2592,12 +2593,12 @@ lpfc_nvme_destroy_localport(struct lpfc_vport *vport)struct lpfc_nvme_lport *lport;struct lpfc_nvme_ctrl_stat *cstat;int ret; +DECLARE_COMPLETION_ONSTACK (lport_unreg_cmp);if (vport->nvmei_support ==0 )return ; localport = vport->localport; - vport->localport =NULL ; lport = (struct lpfc_nvme_lport *)localport->private; cstat = lport->cstat;
@@ -2608,13 +2609,14 @@ lpfc_nvme_destroy_localport(struct lpfc_vport *vport)/* lport's rport list is clear. Unregister * lport and release resources. */ -init_completion (&lport->lport_unreg_done); + lport->lport_unreg_cmp = &lport_unreg_cmp; ret =nvme_fc_unregister_localport (localport);/* Wait for completion. This either blocks * indefinitely or succeeds */ -lpfc_nvme_lport_unreg_wait (vport, lport); +lpfc_nvme_lport_unreg_wait (vport, lport, &lport_unreg_cmp); + vport->localport =NULL ;kfree (cstat);/* Regardless of the unregister upcall response, clear
drivers/scsi/lpfc/lpfc_nvme.h +1 -1
@@ -50,7 +50,7 @@ struct lpfc_nvme_ctrl_stat {/* Declare nvme-based local and remote port definitions. */ struct lpfc_nvme_lport {struct lpfc_vport *vport; -struct completion lport_unreg_done; +struct completion *lport_unreg_cmp;/* Add stats counters here */ struct lpfc_nvme_ctrl_stat *cstat;atomic_t fc4NvmeLsRequests;
Signature · 1 line
2.1.0
PATCH 1/2 lpfc: nvme: avoid hang / use-after-free when destroying localport
Quoted 5 lines · Ewan D. Milne
We cannot wait on a completion object in the
lpfc_nvme_lportstructure in the_destroy_localport()code path because the NVMe/fc transport will free that structure immediately after the .localport_delete() callback. This results in a use-after-free, and a hang if slub_debug=FZPU is enabled.
Quoted 9 lines
Fix this by putting the completion on the stack.
Signed-off-by Ewan D. Milne <emilne at redhat.com>--- drivers/scsi/lpfc/lpfc_nvme.c | 16 +++++++++------- drivers/scsi/lpfc/lpfc_nvme.h | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-)
Thank you Ewan
-- james
PATCH 2/2 lpfc: nvmet: avoid hang / use-after-free when destroying targetport
In reply to Ewan D. MilneWe cannot wait on a completion object in the lpfc_nvme_targetport structure in the _destroy_targetport() code path because the NVMe/fc transport will free that structure immediately after the .targetport_delete() callback. This results in a use-after-free, and a hang if slub_debug=FZPU is enabled.
Fix this by putting the completion on the stack.
--- drivers/scsi/lpfc/lpfc_nvmet.c | 8 +++++--- drivers/scsi/lpfc/lpfc_nvmet.h | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-)
drivers/scsi/lpfc/lpfc_nvmet.c +5 -3
@@ -1003,7 +1003,8 @@ lpfc_nvmet_targetport_delete(struct nvmet_fc_target_port *targetport)struct lpfc_nvmet_tgtport *tport = targetport->private;/* release any threads waiting for the unreg to complete */ -complete (&tport->tport_unreg_done); +if (tport->phba->targetport) +complete (tport->tport_unreg_cmp); } static void
@@ -1692,6 +1693,7 @@ lpfc_nvmet_destroy_targetport(struct lpfc_hba *phba)struct lpfc_nvmet_tgtport *tgtp;struct lpfc_queue *wq;uint32_t qidx; +DECLARE_COMPLETION_ONSTACK (tport_unreg_cmp);if (phba->nvmet_support ==0 )return ;
@@ -1701,9 +1703,9 @@ lpfc_nvmet_destroy_targetport(struct lpfc_hba *phba) wq = phba->sli4_hba.nvme_wq[qidx];lpfc_nvmet_wqfull_flush (phba, wq,NULL ); } -init_completion (&tgtp->tport_unreg_done); + tgtp->tport_unreg_cmp = &tport_unreg_cmp;nvmet_fc_unregister_targetport (phba->targetport); -wait_for_completion_timeout (&tgtp->tport_unreg_done,5 ); +wait_for_completion_timeout (&tport_unreg_cmp,5 );lpfc_nvmet_cleanup_io_context (phba); } phba->targetport =NULL ;
drivers/scsi/lpfc/lpfc_nvmet.h +1 -1
@@ -34,7 +34,7 @@/* Used for NVME Target */ struct lpfc_nvmet_tgtport {struct lpfc_hba *phba; -struct completion tport_unreg_done; +struct completion *tport_unreg_cmp;/* Stats counters - lpfc_nvmet_unsol_ls_buffer */ atomic_t rcv_ls_req_in;
Signature · 1 line
2.1.0
PATCH 2/2 lpfc: nvmet: avoid hang / use-after-free when destroying targetport
Quoted 5 lines · Ewan D. Milne
We cannot wait on a completion object in the
lpfc_nvme_targetportstructure in the_destroy_targetport()code path because the NVMe/fc transport will free that structure immediately after the .targetport_delete() callback. This results in a use-after-free, and a hang if slub_debug=FZPU is enabled.
Quoted 9 lines
Fix this by putting the completion on the stack.
Signed-off-by Ewan D. Milne <emilne at redhat.com>--- drivers/scsi/lpfc/lpfc_nvmet.c | 8 +++++--- drivers/scsi/lpfc/lpfc_nvmet.h | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-)
Thank you Ewan
-- james
Martin K. Petersen
Appliedcover Fix use-after-free errors in lpfc driver when using NVMe
In reply to Ewan D. MilneEwan,
Quoted 7 lines
These two patches fix use-after-free errors in the shutdown path of the lpfc driver with both the Initiator and Target mode usage of FC ports.
The problem is very apparent with
slub_debugenabled, as the memory poisoning prevents thewait_for_completion_timeout()from returning after the object has been freed.
Applied to 5.0/scsi-fixes. Thanks!
Signature · 1 line
Martin K. Petersen Oracle Linux Engineering