scsi: fc_transport: vport and rport cleanup synchronization
PATCH 3/3 scsi: fc_transport: vport and rport cleanup synchronization
Imporve synchronization and cleanup logic in the fc_remove_host() and fc_rport_final_delete() to prevent use-after-free conditions during host removal
Vport cleanup:
- Mark all vports with FC_VPORT_DELETING under lock
- Cancel all vport work synchronously before removing from the list
- Synchronous deletion with fc_vport_terminate()
Rport cleanup, applied to rports and rport_binding
- Mark all rport with FC_PORTSTATE_DELETED under lock
- Cancel all timers and work synchronously before removing from the list
- Call fc_rport_final_delete() synchronously instead of queuing
fc_rport_final_delete():
- Calling cancel_delayed_work_sync() for any outstanding delayed work
- Clear FC_RPORT_DEVLOSS_PENDING under lock
- Flushing all pending work completes before destruction
--- drivers/scsi/scsi_transport_fc.c | 85 ++++++++++++++++++++++++-------- 1 file changed, 64 insertions(+), 21 deletions(-)
drivers/scsi/scsi_transport_fc.c +0 -0
@@ -39,6 +39,7 @@ static void fc_li_stats_update(u16 event_type,static void fc_delivery_stats_update (u32 reason_code,struct fc_fpin_stats *stats); static void fc_cn_stats_update(u16 event_type, struct fc_fpin_stats
*stats);
+static void fc_rport_final_delete(struct work_struct *work);
/*
* Module Parameters
@@ -2883,31 +2884,71 @@ fc_remove_host(struct Scsi_Host *shost)
struct fc_host_attrs *fc_host = shost_to_fc_host(shost); unsigned long flags;
- spin_lock_irqsave(shost->host_lock, flags);
-
/* Remove any vports */
+ /* Mark FC_VPORT_DELETING for now */
+ spin_lock_irqsave(shost->host_lock, flags);
list_for_each_entry_safe(vport, next_vport, &fc_host->vports, peers) {
vport->flags |= FC_VPORT_DELETING;
- fc_queue_work(shost, &vport->vport_delete_work);
+ }
+ spin_unlock_irqrestore(shost->host_lock, flags);
+
+ /*
+ * remove all vport works synchronously BEFORE removing from list.
+ * This prevents use-after-free when timers fire.
+ */
+ list_for_each_entry_safe(vport, next_vport, &fc_host->vports, peers) {
+ /* Cancel any pending work/timers */
+ cancel_work_sync(&vport->vport_delete_work);
+ /* Now safe to do synchronous deletion */
+ fc_vport_terminate(vport);
}
/* Remove any remote ports */
+ /* Mark rports and rport_bindings with FC_PORTSTATE_DELETED for now */
+ spin_lock_irqsave(shost->host_lock, flags);
list_for_each_entry_safe(rport, next_rport,
&fc_host->rports, peers) {
- list_del(&rport->peers);
rport->port_state = FC_PORTSTATE_DELETED;
- fc_queue_work(shost, &rport->rport_delete_work);
}
list_for_each_entry_safe(rport, next_rport,
&fc_host->rport_bindings, peers) {
- list_del(&rport->peers);
rport->port_state = FC_PORTSTATE_DELETED;
- fc_queue_work(shost, &rport->rport_delete_work);
}
-
spin_unlock_irqrestore(shost->host_lock, flags);
+ list_for_each_entry_safe(rport, next_rport,
+ &fc_host->rports, peers) {
+ /* Cancel ALL timers and work before removing from list */
+ cancel_delayed_work_sync(&rport->fail_io_work);
+ cancel_delayed_work_sync(&rport->dev_loss_work);
+ cancel_work_sync(&rport->scan_work);
+ cancel_work_sync(&rport->stgt_delete_work);
+
+ spin_lock_irqsave(shost->host_lock, flags);
+ list_del(&rport->peers);
+ spin_unlock_irqrestore(shost->host_lock, flags);
+
+ /* Now safe to do final deletion synchronously */
+ fc_rport_final_delete(&rport->rport_delete_work);
+ }
+
+ list_for_each_entry_safe(rport, next_rport,
+ &fc_host->rport_bindings, peers) {
+ /* Cancel ALL timers and work before removing from list */
+ cancel_delayed_work_sync(&rport->fail_io_work);
+ cancel_delayed_work_sync(&rport->dev_loss_work);
+ cancel_work_sync(&rport->scan_work);
+ cancel_work_sync(&rport->stgt_delete_work);
+
+ spin_lock_irqsave(shost->host_lock, flags);
+ list_del(&rport->peers);
+ spin_unlock_irqrestore(shost->host_lock, flags);
+
+ /* Now safe to do final deletion synchronously */
+ fc_rport_final_delete(&rport->rport_delete_work);
+ }
+
/* flush all scan work items */
scsi_flush_work(shost);
@@ -2983,21 +3024,22 @@ fc_rport_final_delete(struct work_struct *work)
scsi_flush_work(shost);
/*
- * Cancel any outstanding timers. These should really exist
- * only when rmmod'ing the LLDD and we're asking for
- * immediate termination of the rports
+ * Cancel any outstanding delayed work synchronously.
+ * This must be done BEFORE taking spinlock and BEFORE
+ * any state changes, as cancel_delayed_work_sync() can sleep.
+ *
+ * These timers should only exist when rmmod'ing the LLDD
+ * and we're asking for immediate termination of rports.
+ */
+ cancel_delayed_work_sync(&rport->fail_io_work);
+ cancel_delayed_work_sync(&rport->dev_loss_work);
+ cancel_work_sync(&rport->scan_work);
+ /*
+ * Now safe to clear the flag under spinlock since all
+ * async work has been cancelled.
*/
spin_lock_irqsave(shost->host_lock, flags);
- if (rport->flags & FC_RPORT_DEVLOSS_PENDING) {
- spin_unlock_irqrestore(shost->host_lock, flags);
- if (!cancel_delayed_work(&rport->fail_io_work))
- fc_flush_devloss(shost, rport);
- if (!cancel_delayed_work(&rport->dev_loss_work))
- fc_flush_devloss(shost, rport);
- cancel_work_sync(&rport->scan_work);
- spin_lock_irqsave(shost->host_lock, flags);
- rport->flags &= ~FC_RPORT_DEVLOSS_PENDING;
- }
+ rport->flags &= ~FC_RPORT_DEVLOSS_PENDING;
spin_unlock_irqrestore(shost->host_lock, flags);
/* Delete SCSI target and sdevs */
@@ -3027,6 +3069,7 @@ fc_rport_final_delete(struct work_struct *work)
if (rport->devloss_work_q) {
work_q = rport->devloss_work_q; rport->devloss_work_q = NULL;
+ flush_workqueue(work_q);
destroy_workqueue(work_q);
}
Signature · 1 line
2.52.0
Quoted 156 lines · Kyle Mahlkuch
Imporve synchronization and cleanup logic in the
fc_remove_host()andfc_rport_final_delete()to prevent use-after-free conditions during host removalVport cleanup:
- Mark all vports withFC_VPORT_DELETINGunder lock
- Cancel all vport work synchronously before removing from the list
- Synchronous deletion withfc_vport_terminate()Rport cleanup, applied to rports and
rport_binding
- Mark all rport withFC_PORTSTATE_DELETEDunder lock
- Cancel all timers and work synchronously before removing from the list
- Callfc_rport_final_delete()synchronously instead of queuing
fc_rport_final_delete():
- Callingcancel_delayed_work_sync()for any outstanding delayed work
- ClearFC_RPORT_DEVLOSS_PENDINGunder lock
- Flushing all pending work completes before destructionSigned-off-by Thinh Tran <[email protected]>Signed-off-by Kyle Mahlkuch <[email protected]>--- drivers/scsi/scsi_transport_fc.c | 85 ++++++++++++++++++++++++-------- 1 file changed, 64 insertions(+), 21 deletions(-)drivers/scsi/scsi_transport_fc.c +1
-0@@ -39,6 +39,7 @@ static void fc_li_stats_update(u16 event_type, static void fc_delivery_stats_update(u32 reason_code, struct fc_fpin_stats *stats); static void fc_cn_stats_update(u16 event_type, struct fc_fpin_stats *stats); +static void fc_rport_final_delete (struct work_struct *work); /** Module Parameters
drivers/scsi/scsi_transport_fc.c +62
-21@@ -2883,31 +2884,71 @@ fc_remove_host(struct Scsi_Host *shost) struct fc_host_attrs *fc_host = shost_to_fc_host(shost); unsigned long flags; - spin_lock_irqsave(shost->host_lock, flags); - /* Remove any vports */ + /* Mark FC_VPORT_DELETING for now */ + spin_lock_irqsave(shost->host_lock, flags); list_for_each_entry_safe(vport, next_vport, &fc_host->vports, peers) { vport->flags |= FC_VPORT_DELETING; - fc_queue_work(shost, &vport->vport_delete_work); + } + spin_unlock_irqrestore(shost->host_lock, flags); + + /* + * remove all vport works synchronously BEFORE removing from list. + * This prevents use-after-free when timers fire. + */ + list_for_each_entry_safe(vport, next_vport, &fc_host->vports, peers) { + /* Cancel any pending work/timers */ + cancel_work_sync(&vport->vport_delete_work); + /* Now safe to do synchronous deletion */ + fc_vport_terminate(vport); } /* Remove any remote ports */ + /* Mark rports and rport_bindings with FC_PORTSTATE_DELETED for now */ + spin_lock_irqsave(shost->host_lock, flags); list_for_each_entry_safe(rport, next_rport, &fc_host->rports, peers) { - list_del(&rport->peers); rport->port_state = FC_PORTSTATE_DELETED; - fc_queue_work(shost, &rport->rport_delete_work); } list_for_each_entry_safe(rport, next_rport, &fc_host->rport_bindings, peers) { - list_del(&rport->peers); rport->port_state = FC_PORTSTATE_DELETED; - fc_queue_work(shost, &rport->rport_delete_work); } - spin_unlock_irqrestore(shost->host_lock, flags); + list_for_each_entry_safe(rport, next_rport, + &fc_host->rports, peers) { + /* Cancel ALL timers and work before removing from list */ + cancel_delayed_work_sync(&rport->fail_io_work); + cancel_delayed_work_sync(&rport->dev_loss_work); + cancel_work_sync(&rport->scan_work); + cancel_work_sync(&rport->stgt_delete_work); + + spin_lock_irqsave(shost->host_lock, flags); + list_del(&rport->peers); + spin_unlock_irqrestore(shost->host_lock, flags); + + /* Now safe to do final deletion synchronously */ + fc_rport_final_delete(&rport->rport_delete_work); + } + + list_for_each_entry_safe(rport, next_rport, + &fc_host->rport_bindings, peers) { + /* Cancel ALL timers and work before removing from list */ + cancel_delayed_work_sync(&rport->fail_io_work); + cancel_delayed_work_sync(&rport->dev_loss_work); + cancel_work_sync(&rport->scan_work); + cancel_work_sync(&rport->stgt_delete_work); + + spin_lock_irqsave(shost->host_lock, flags); + list_del(&rport->peers); + spin_unlock_irqrestore(shost->host_lock, flags); + + /* Now safe to do final deletion synchronously */ + fc_rport_final_delete(&rport->rport_delete_work); + } + /* flush all scan work items */ scsi_flush_work(shost);@@ -2983,21 +3024,22 @@ fc_rport_final_delete(struct work_struct *work) scsi_flush_work(shost); /* - * Cancel any outstanding timers. These should really exist - * only when rmmod'ing the LLDD and we're asking for - * immediate termination of the rports + * Cancel any outstanding delayed work synchronously. + * This must be done BEFORE taking spinlock and BEFORE + * any state changes, as cancel_delayed_work_sync() can sleep. + * + * These timers should only exist when rmmod'ing the LLDD + * and we're asking for immediate termination of rports. + */ + cancel_delayed_work_sync(&rport->fail_io_work); + cancel_delayed_work_sync(&rport->dev_loss_work); + cancel_work_sync(&rport->scan_work); + /* + * Now safe to clear the flag under spinlock since all + * async work has been cancelled. */ spin_lock_irqsave(shost->host_lock, flags); - if (rport->flags & FC_RPORT_DEVLOSS_PENDING) { - spin_unlock_irqrestore(shost->host_lock, flags); - if (!cancel_delayed_work(&rport->fail_io_work)) - fc_flush_devloss(shost, rport); - if (!cancel_delayed_work(&rport->dev_loss_work)) - fc_flush_devloss(shost, rport); - cancel_work_sync(&rport->scan_work); - spin_lock_irqsave(shost->host_lock, flags); - rport->flags &= ~FC_RPORT_DEVLOSS_PENDING; - } + rport->flags &= ~FC_RPORT_DEVLOSS_PENDING; spin_unlock_irqrestore(shost->host_lock, flags); /* Delete SCSI target and sdevs */
drivers/scsi/scsi_transport_fc.c fc_rport_final_delete(struct work_struct *work) · line 3075
Quoted 7 lines
drivers/scsi/scsi_transport_fc.c +1
-0@@ -3027,6 +3069,7 @@ fc_rport_final_delete(struct work_struct *work) if (rport->devloss_work_q) { work_q = rport->devloss_work_q; rport->devloss_work_q = NULL; + flush_workqueue(work_q); destroy_workqueue(work_q); }
Similar remarks to the previous patch.
If we set 'SHOST_CANCEL' after we scheduled all these elements and before calling scsi_flush_work() we can inhibit further calls to fc_queue_work() and we won't need this.
Cheers,
Hannes
Signature · 4 lines
Dr. Hannes Reinecke Kernel Storage Architect [email protected] +49 911 74053 688 SUSE Software Solutions GmbH, Frankenstr. 146, 90461 Nürnberg HRB 36809 (AG Nürnberg), GF: I. Totev, A. McDonald, W. Knoblich
Quoted 14 lines · Kyle Mahlkuch
Imporve synchronization and cleanup logic in the
fc_remove_host()andfc_rport_final_delete()to prevent use-after-free conditions during host removalVport cleanup:
- Mark all vports withFC_VPORT_DELETINGunder lock
- Cancel all vport work synchronously before removing from the list
- Synchronous deletion withfc_vport_terminate()Rport cleanup, applied to rports and
rport_binding
- Mark all rport withFC_PORTSTATE_DELETEDunder lock
- Cancel all timers and work synchronously before removing from the list
- Callfc_rport_final_delete()synchronously instead of queuing
Quoted 8 lines
fc_rport_final_delete():
- Callingcancel_delayed_work_sync()for any outstanding delayed work
- ClearFC_RPORT_DEVLOSS_PENDINGunder lock
- Flushing all pending work completes before destructionSigned-off-by Thinh Tran <[email protected]>Signed-off-by Kyle Mahlkuch <[email protected]>---
Tested the complete 3-patch series on an IBM Power11 (ppc64le) server equipped with an Emulex FC HBA. The patches applied cleanly, and FC driver parameter validation testing was executed continuously for 36 hours. No crashes, hangs, or functional issues were observed during the test period.
Thanks,
Maram Srimannarayana Murthy
Quoted 141 lines
drivers/scsi/scsi_transport_fc.c | 85 ++++++++++++++++++++++++--------
1 file changed, 64 insertions(+), 21 deletions(-)drivers/scsi/scsi_transport_fc.c +16
-3@@ -39,6 +39,7 @@ static void fc_li_stats_update(u16 event_type, static void fc_delivery_stats_update(u32 reason_code, struct fc_fpin_stats *stats); static void fc_cn_stats_update(u16 event_type, struct fc_fpin_stats *stats); +static void fc_rport_final_delete (struct work_struct *work); /** Module Parameters @@ -2883,31 +2884,71 @@ fc_remove_host(struct Scsi_Host *shost) struct fc_host_attrs *fc_host = shost_to_fc_host(shost); unsigned long flags; - spin_lock_irqsave(shost->host_lock, flags); - /* Remove any vports */ + /* Mark FC_VPORT_DELETING for now */ + spin_lock_irqsave(shost->host_lock, flags); list_for_each_entry_safe(vport, next_vport, &fc_host->vports, peers) { vport->flags |= FC_VPORT_DELETING; - fc_queue_work(shost, &vport->vport_delete_work); + } + spin_unlock_irqrestore(shost->host_lock, flags); + + /* + * remove all vport works synchronously BEFORE removing from list. + * This prevents use-after-free when timers fire. + */ + list_for_each_entry_safe(vport, next_vport, &fc_host->vports, peers) { + /* Cancel any pending work/timers */ + cancel_work_sync(&vport->vport_delete_work); + /* Now safe to do synchronous deletion */ + fc_vport_terminate(vport); } /* Remove any remote ports */ + /* Mark rports and rport_bindings with FC_PORTSTATE_DELETED fornow */
+spin_lock_irqsave(shost->host_lock, flags);list_for_each_entry_safe(rport, next_rport, &fc_host->rports, peers){
-list_del(&rport->peers);rport->port_state=FC_PORTSTATE_DELETED; -fc_queue_work(shost, &rport->rport_delete_work); }
list_for_each_entry_safe(rport, next_rport, &fc_host->rport_bindings, peers){ -list_del(&rport->peers);rport->port_state=FC_PORTSTATE_DELETED; -fc_queue_work(shost, &rport->rport_delete_work); }
-
spin_unlock_irqrestore(shost->host_lock, flags);+ list_for_each_entry_safe(rport,
next_rport,
+&fc_host->rports, peers) {
+ /* Cancel ALL timers and work before removing from list */ +cancel_delayed_work_sync(&rport->fail_io_work); +cancel_delayed_work_sync(&rport->dev_loss_work); +cancel_work_sync(&rport->scan_work);
+cancel_work_sync(&rport->stgt_delete_work);drivers/scsi/scsi_transport_fc.c +25
-0+ + spin_lock_irqsave(shost->host_lock, flags); + list_del(&rport->peers); + spin_unlock_irqrestore(shost->host_lock, flags); + + /* Now safe to do final deletion synchronously */ + fc_rport_final_delete(&rport->rport_delete_work); + } + + list_for_each_entry_safe(rport, next_rport, + &fc_host->rport_bindings, peers) { + /* Cancel ALL timers and work before removing from list */ + cancel_delayed_work_sync(&rport->fail_io_work); + cancel_delayed_work_sync(&rport->dev_loss_work); + cancel_work_sync(&rport->scan_work); + cancel_work_sync(&rport->stgt_delete_work); + + spin_lock_irqsave(shost->host_lock, flags); + list_del(&rport->peers); + spin_unlock_irqrestore(shost->host_lock, flags); + + /* Now safe to do final deletion synchronously */ + fc_rport_final_delete(&rport->rport_delete_work); + } +/* flush all scan work items */
scsi_flush_work(shost);drivers/scsi/scsi_transport_fc.c +15
-13@@ -2983,21 +3024,22 @@ fc_rport_final_delete(struct work_struct *work) scsi_flush_work(shost); /* - * Cancel any outstanding timers. These should really exist - * only when rmmod'ing the LLDD and we're asking for - * immediate termination of the rports + * Cancel any outstanding delayed work synchronously. + * This must be done BEFORE taking spinlock and BEFORE + * any state changes, as cancel_delayed_work_sync() can sleep. + * + * These timers should only exist when rmmod'ing the LLDD + * and we're asking for immediate termination of rports. + */ + cancel_delayed_work_sync(&rport->fail_io_work); + cancel_delayed_work_sync(&rport->dev_loss_work); + cancel_work_sync(&rport->scan_work); + /* + * Now safe to clear the flag under spinlock since all + * async work has been cancelled. */ spin_lock_irqsave(shost->host_lock, flags); - if (rport->flags & FC_RPORT_DEVLOSS_PENDING) { - spin_unlock_irqrestore(shost->host_lock, flags); - if (!cancel_delayed_work(&rport->fail_io_work)) - fc_flush_devloss(shost, rport); - if (!cancel_delayed_work(&rport->dev_loss_work)) - fc_flush_devloss(shost, rport); - cancel_work_sync(&rport->scan_work); - spin_lock_irqsave(shost->host_lock, flags); - rport->flags &= ~FC_RPORT_DEVLOSS_PENDING; - } + rport->flags &= ~FC_RPORT_DEVLOSS_PENDING; spin_unlock_irqrestore(shost->host_lock, flags); /* Delete SCSI target and sdevs */@@ -3027,6 +3069,7 @@ fc_rport_final_delete(struct work_struct *work) if (rport->devloss_work_q) { work_q = rport->devloss_work_q; rport->devloss_work_q = NULL; + flush_workqueue(work_q); destroy_workqueue(work_q); }