Skip to contentSkip to conversation

scsi: fc_transport: vport and rport cleanup synchronization

PATCH Superseded Kyle Mahlkuch 3 participants 3 messages Apr 9 – Sep 1 SCSI

v1 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

PATCH 3/3 scsi: fc_transport: vport and rport cleanup synchronization

Imporve synchronization and cleanup logic in the fc_remove_host() and fc_rport_final_delete() to prevent use-after-free conditions during host removal

Vport cleanup:
- Mark all vports with FC_VPORT_DELETING under lock
- Cancel all vport work synchronously before removing from the list
- Synchronous deletion with fc_vport_terminate()

Rport cleanup, applied to rports and rport_binding
- Mark all rport with FC_PORTSTATE_DELETED under lock
- Cancel all timers and work synchronously before removing from the list
- Call fc_rport_final_delete() synchronously instead of queuing

fc_rport_final_delete():
- Calling cancel_delayed_work_sync() for any outstanding delayed work
- Clear FC_RPORT_DEVLOSS_PENDING under lock
- Flushing all pending work completes before destruction

---
 drivers/scsi/scsi_transport_fc.c | 85 ++++++++++++++++++++++++--------
 1 file changed, 64 insertions(+), 21 deletions(-)
drivers/scsi/scsi_transport_fc.c +0 -0
index 123b22b52640..0adb9330befc 100644
@@ -39,6 +39,7 @@ static void fc_li_stats_update(u16 event_type,
 static void fc_delivery_stats_update(u32 reason_code,
 				     struct fc_fpin_stats *stats);
 static void fc_cn_stats_update(u16 event_type, struct fc_fpin_stats 

*stats);
+static void fc_rport_final_delete(struct work_struct *work);

/*
* Module Parameters
@@ -2883,31 +2884,71 @@ fc_remove_host(struct Scsi_Host *shost)
struct fc_host_attrs *fc_host = shost_to_fc_host(shost); unsigned long flags;

- spin_lock_irqsave(shost->host_lock, flags);
-
/* Remove any vports */
+ /* Mark FC_VPORT_DELETING for now */
+ spin_lock_irqsave(shost->host_lock, flags);
list_for_each_entry_safe(vport, next_vport, &fc_host->vports, peers) {
vport->flags |= FC_VPORT_DELETING;
- fc_queue_work(shost, &vport->vport_delete_work);
+ }
+ spin_unlock_irqrestore(shost->host_lock, flags);
+
+ /*
+ * remove all vport works synchronously BEFORE removing from list.
+ * This prevents use-after-free when timers fire.
+ */
+ list_for_each_entry_safe(vport, next_vport, &fc_host->vports, peers) {
+ /* Cancel any pending work/timers */
+ cancel_work_sync(&vport->vport_delete_work);
+ /* Now safe to do synchronous deletion */
+ fc_vport_terminate(vport);
}

/* Remove any remote ports */
+ /* Mark rports and rport_bindings with FC_PORTSTATE_DELETED for now */
+ spin_lock_irqsave(shost->host_lock, flags);
list_for_each_entry_safe(rport, next_rport,
&fc_host->rports, peers) {
- list_del(&rport->peers);
rport->port_state = FC_PORTSTATE_DELETED;
- fc_queue_work(shost, &rport->rport_delete_work);
}

list_for_each_entry_safe(rport, next_rport,
&fc_host->rport_bindings, peers) {
- list_del(&rport->peers);
rport->port_state = FC_PORTSTATE_DELETED;
- fc_queue_work(shost, &rport->rport_delete_work);
}
-
spin_unlock_irqrestore(shost->host_lock, flags);

+ list_for_each_entry_safe(rport, next_rport,
+ &fc_host->rports, peers) {
+ /* Cancel ALL timers and work before removing from list */
+ cancel_delayed_work_sync(&rport->fail_io_work);
+ cancel_delayed_work_sync(&rport->dev_loss_work);
+ cancel_work_sync(&rport->scan_work);
+ cancel_work_sync(&rport->stgt_delete_work);
+
+ spin_lock_irqsave(shost->host_lock, flags);
+ list_del(&rport->peers);
+ spin_unlock_irqrestore(shost->host_lock, flags);
+
+ /* Now safe to do final deletion synchronously */
+ fc_rport_final_delete(&rport->rport_delete_work);
+ }
+
+ list_for_each_entry_safe(rport, next_rport,
+ &fc_host->rport_bindings, peers) {
+ /* Cancel ALL timers and work before removing from list */
+ cancel_delayed_work_sync(&rport->fail_io_work);
+ cancel_delayed_work_sync(&rport->dev_loss_work);
+ cancel_work_sync(&rport->scan_work);
+ cancel_work_sync(&rport->stgt_delete_work);
+
+ spin_lock_irqsave(shost->host_lock, flags);
+ list_del(&rport->peers);
+ spin_unlock_irqrestore(shost->host_lock, flags);
+
+ /* Now safe to do final deletion synchronously */
+ fc_rport_final_delete(&rport->rport_delete_work);
+ }
+
/* flush all scan work items */
scsi_flush_work(shost);

@@ -2983,21 +3024,22 @@ fc_rport_final_delete(struct work_struct *work)
scsi_flush_work(shost);

/*
- * Cancel any outstanding timers. These should really exist
- * only when rmmod'ing the LLDD and we're asking for
- * immediate termination of the rports
+ * Cancel any outstanding delayed work synchronously.
+ * This must be done BEFORE taking spinlock and BEFORE
+ * any state changes, as cancel_delayed_work_sync() can sleep.
+ *
+ * These timers should only exist when rmmod'ing the LLDD
+ * and we're asking for immediate termination of rports.
+ */
+ cancel_delayed_work_sync(&rport->fail_io_work);
+ cancel_delayed_work_sync(&rport->dev_loss_work);
+ cancel_work_sync(&rport->scan_work);
+ /*
+ * Now safe to clear the flag under spinlock since all
+ * async work has been cancelled.
*/
spin_lock_irqsave(shost->host_lock, flags);
- if (rport->flags & FC_RPORT_DEVLOSS_PENDING) {
- spin_unlock_irqrestore(shost->host_lock, flags);
- if (!cancel_delayed_work(&rport->fail_io_work))
- fc_flush_devloss(shost, rport);
- if (!cancel_delayed_work(&rport->dev_loss_work))
- fc_flush_devloss(shost, rport);
- cancel_work_sync(&rport->scan_work);
- spin_lock_irqsave(shost->host_lock, flags);
- rport->flags &= ~FC_RPORT_DEVLOSS_PENDING;
- }
+ rport->flags &= ~FC_RPORT_DEVLOSS_PENDING;
spin_unlock_irqrestore(shost->host_lock, flags);

/* Delete SCSI target and sdevs */
@@ -3027,6 +3069,7 @@ fc_rport_final_delete(struct work_struct *work)
if (rport->devloss_work_q) {
work_q = rport->devloss_work_q; rport->devloss_work_q = NULL;
+ flush_workqueue(work_q);
destroy_workqueue(work_q);
}

Signature · 1 line
2.52.0
#1 lore raw
Quoted 156 lines · Kyle Mahlkuch

Imporve synchronization and cleanup logic in the fc_remove_host() and fc_rport_final_delete() to prevent use-after-free conditions during host removal

Vport cleanup:
  - Mark all vports with FC_VPORT_DELETING under lock
  - Cancel all vport work synchronously before removing from the list
  - Synchronous deletion with fc_vport_terminate()

Rport cleanup, applied to rports and rport_binding
  - Mark all rport with FC_PORTSTATE_DELETED under lock
  - Cancel all timers and work synchronously before removing from the list
  - Call fc_rport_final_delete() synchronously instead of queuing

fc_rport_final_delete():
  - Calling cancel_delayed_work_sync() for any outstanding delayed work
  - Clear FC_RPORT_DEVLOSS_PENDING under lock
  - Flushing all pending work completes before destruction

---
  drivers/scsi/scsi_transport_fc.c | 85 ++++++++++++++++++++++++--------
  1 file changed, 64 insertions(+), 21 deletions(-)
drivers/scsi/scsi_transport_fc.c +1 -0
index 123b22b52640..0adb9330befc 100644
@@ -39,6 +39,7 @@ static void fc_li_stats_update(u16 event_type,
  static void fc_delivery_stats_update(u32 reason_code,
                       struct fc_fpin_stats *stats);
  static void fc_cn_stats_update(u16 event_type, struct fc_fpin_stats 
 *stats);
+static void fc_rport_final_delete(struct work_struct *work);
 
  /*

  * Module Parameters

drivers/scsi/scsi_transport_fc.c +62 -21
@@ -2883,31 +2884,71 @@ fc_remove_host(struct Scsi_Host *shost)
      struct fc_host_attrs *fc_host = shost_to_fc_host(shost);
      unsigned long flags;
 
-    spin_lock_irqsave(shost->host_lock, flags);
-
      /* Remove any vports */
+    /* Mark FC_VPORT_DELETING for now */
+    spin_lock_irqsave(shost->host_lock, flags);
      list_for_each_entry_safe(vport, next_vport, &fc_host->vports, 
 peers) {
          vport->flags |= FC_VPORT_DELETING;
-        fc_queue_work(shost, &vport->vport_delete_work);
+    }
+    spin_unlock_irqrestore(shost->host_lock, flags);
+
+    /*
+     * remove all vport works synchronously BEFORE removing from list.
+     * This prevents use-after-free when timers fire.
+     */
+    list_for_each_entry_safe(vport, next_vport, &fc_host->vports, peers) {
+        /* Cancel any pending work/timers */
+        cancel_work_sync(&vport->vport_delete_work);
+        /* Now safe to do synchronous deletion */
+        fc_vport_terminate(vport);
      }
 
      /* Remove any remote ports */
+    /* Mark rports and rport_bindings with FC_PORTSTATE_DELETED for now */
+    spin_lock_irqsave(shost->host_lock, flags);
      list_for_each_entry_safe(rport, next_rport,
              &fc_host->rports, peers) {
-        list_del(&rport->peers);
          rport->port_state = FC_PORTSTATE_DELETED;
-        fc_queue_work(shost, &rport->rport_delete_work);
      }
 
      list_for_each_entry_safe(rport, next_rport,
              &fc_host->rport_bindings, peers) {
-        list_del(&rport->peers);
          rport->port_state = FC_PORTSTATE_DELETED;
-        fc_queue_work(shost, &rport->rport_delete_work);
      }
-
      spin_unlock_irqrestore(shost->host_lock, flags);
 
+    list_for_each_entry_safe(rport, next_rport,
+            &fc_host->rports, peers) {
+        /* Cancel ALL timers and work before removing from list */
+        cancel_delayed_work_sync(&rport->fail_io_work);
+        cancel_delayed_work_sync(&rport->dev_loss_work);
+        cancel_work_sync(&rport->scan_work);
+        cancel_work_sync(&rport->stgt_delete_work);
+
+        spin_lock_irqsave(shost->host_lock, flags);
+        list_del(&rport->peers);
+        spin_unlock_irqrestore(shost->host_lock, flags);
+
+        /* Now safe to do final deletion synchronously */
+        fc_rport_final_delete(&rport->rport_delete_work);
+    }
+
+    list_for_each_entry_safe(rport, next_rport,
+            &fc_host->rport_bindings, peers) {
+        /* Cancel ALL timers and work before removing from list */
+        cancel_delayed_work_sync(&rport->fail_io_work);
+        cancel_delayed_work_sync(&rport->dev_loss_work);
+        cancel_work_sync(&rport->scan_work);
+        cancel_work_sync(&rport->stgt_delete_work);
+
+        spin_lock_irqsave(shost->host_lock, flags);
+        list_del(&rport->peers);
+        spin_unlock_irqrestore(shost->host_lock, flags);
+
+        /* Now safe to do final deletion synchronously */
+        fc_rport_final_delete(&rport->rport_delete_work);
+    }
+
      /* flush all scan work items */
      scsi_flush_work(shost);
@@ -2983,21 +3024,22 @@ fc_rport_final_delete(struct work_struct *work)
          scsi_flush_work(shost);
 
      /*
-     * Cancel any outstanding timers. These should really exist
-     * only when rmmod'ing the LLDD and we're asking for
-     * immediate termination of the rports
+     * Cancel any outstanding delayed work synchronously.
+     * This must be done BEFORE taking spinlock and BEFORE
+     * any state changes, as cancel_delayed_work_sync() can sleep.
+     *
+     * These timers should only exist when rmmod'ing the LLDD
+     * and we're asking for immediate termination of rports.
+     */
+    cancel_delayed_work_sync(&rport->fail_io_work);
+    cancel_delayed_work_sync(&rport->dev_loss_work);
+    cancel_work_sync(&rport->scan_work);
+    /*
+     * Now safe to clear the flag under spinlock since all
+     * async work has been cancelled.
       */
      spin_lock_irqsave(shost->host_lock, flags);
-    if (rport->flags & FC_RPORT_DEVLOSS_PENDING) {
-        spin_unlock_irqrestore(shost->host_lock, flags);
-        if (!cancel_delayed_work(&rport->fail_io_work))
-            fc_flush_devloss(shost, rport);
-        if (!cancel_delayed_work(&rport->dev_loss_work))
-            fc_flush_devloss(shost, rport);
-        cancel_work_sync(&rport->scan_work);
-        spin_lock_irqsave(shost->host_lock, flags);
-        rport->flags &= ~FC_RPORT_DEVLOSS_PENDING;
-    }
+    rport->flags &= ~FC_RPORT_DEVLOSS_PENDING;
      spin_unlock_irqrestore(shost->host_lock, flags);
 
      /* Delete SCSI target and sdevs */

drivers/scsi/scsi_transport_fc.c fc_rport_final_delete(struct work_struct *work) · line 3075

Quoted 7 lines
drivers/scsi/scsi_transport_fc.c +1 -0
@@ -3027,6 +3069,7 @@ fc_rport_final_delete(struct work_struct *work)
      if (rport->devloss_work_q) {
          work_q = rport->devloss_work_q;
          rport->devloss_work_q = NULL;
+        flush_workqueue(work_q);
          destroy_workqueue(work_q);
      }
 

Similar remarks to the previous patch.
If we set 'SHOST_CANCEL' after we scheduled all these elements and before calling scsi_flush_work() we can inhibit further calls to fc_queue_work() and we won't need this.

Cheers,

Hannes

Signature · 4 lines
Dr. Hannes Reinecke                  Kernel Storage Architect
[email protected]                                +49 911 74053 688
SUSE Software Solutions GmbH, Frankenstr. 146, 90461 Nürnberg
HRB 36809 (AG Nürnberg), GF: I. Totev, A. McDonald, W. Knoblich
Tested-by on v1 3/3
#2 lore raw
In reply to Kyle Mahlkuch
Quoted 14 lines · Kyle Mahlkuch

Imporve synchronization and cleanup logic in the fc_remove_host() and fc_rport_final_delete() to prevent use-after-free conditions during host removal

Vport cleanup:
  - Mark all vports with FC_VPORT_DELETING under lock
  - Cancel all vport work synchronously before removing from the list
  - Synchronous deletion with fc_vport_terminate()

Rport cleanup, applied to rports and rport_binding
  - Mark all rport with FC_PORTSTATE_DELETED under lock
  - Cancel all timers and work synchronously before removing from the list
  - Call fc_rport_final_delete() synchronously instead of queuing

Quoted 8 lines

fc_rport_final_delete():
  - Calling cancel_delayed_work_sync() for any outstanding delayed work
  - Clear FC_RPORT_DEVLOSS_PENDING under lock
  - Flushing all pending work completes before destruction

---

Tested the complete 3-patch series on an IBM Power11 (ppc64le) server equipped with an Emulex FC HBA. The patches applied cleanly, and FC driver parameter validation testing was executed continuously for 36 hours. No crashes, hangs, or functional issues were observed during the test period.

Thanks,
Maram Srimannarayana Murthy

Quoted 141 lines

 drivers/scsi/scsi_transport_fc.c | 85 ++++++++++++++++++++++++--------

 1 file changed, 64 insertions(+), 21 deletions(-)
drivers/scsi/scsi_transport_fc.c +16 -3
index 123b22b52640..0adb9330befc 100644
@@ -39,6 +39,7 @@ static void fc_li_stats_update(u16 event_type,
  static void fc_delivery_stats_update(u32 reason_code,
                       struct fc_fpin_stats *stats);
  static void fc_cn_stats_update(u16 event_type, struct fc_fpin_stats *stats);
+static void fc_rport_final_delete(struct work_struct *work);
 
  /*
   * Module Parameters
@@ -2883,31 +2884,71 @@ fc_remove_host(struct Scsi_Host *shost)
      struct fc_host_attrs *fc_host = shost_to_fc_host(shost);
      unsigned long flags;
 
-    spin_lock_irqsave(shost->host_lock, flags);
-
      /* Remove any vports */
+    /* Mark FC_VPORT_DELETING for now */
+    spin_lock_irqsave(shost->host_lock, flags);
      list_for_each_entry_safe(vport, next_vport, &fc_host->vports, peers) {
          vport->flags |= FC_VPORT_DELETING;
-        fc_queue_work(shost, &vport->vport_delete_work);
+    }
+    spin_unlock_irqrestore(shost->host_lock, flags);
+
+    /*
+     * remove all vport works synchronously BEFORE removing from list.
+     * This prevents use-after-free when timers fire.
+     */
+    list_for_each_entry_safe(vport, next_vport, &fc_host->vports, 
 peers) {
+        /* Cancel any pending work/timers */
+        cancel_work_sync(&vport->vport_delete_work);
+        /* Now safe to do synchronous deletion */
+        fc_vport_terminate(vport);
      }
 
      /* Remove any remote ports */
+    /* Mark rports and rport_bindings with FC_PORTSTATE_DELETED for 

now */
+    spin_lock_irqsave(shost->host_lock, flags); list_for_each_entry_safe(rport, next_rport, &fc_host->rports, peers) {
-        list_del(&rport->peers); rport->port_state = FC_PORTSTATE_DELETED; -        fc_queue_work(shost, &rport->rport_delete_work); }

     list_for_each_entry_safe(rport, next_rport, &fc_host->rport_bindings, peers) { -        list_del(&rport->peers); rport->port_state = FC_PORTSTATE_DELETED; -        fc_queue_work(shost, &rport->rport_delete_work); }
-
     spin_unlock_irqrestore(shost->host_lock, flags);

+    list_for_each_entry_safe(rport, next_rport,
+            &fc_host->rports, peers) {
+        /* Cancel ALL timers and work before removing from list */ +        cancel_delayed_work_sync(&rport->fail_io_work); +        cancel_delayed_work_sync(&rport->dev_loss_work); +        cancel_work_sync(&rport->scan_work);
+        cancel_work_sync(&rport->stgt_delete_work);

drivers/scsi/scsi_transport_fc.c +25 -0
+
+        spin_lock_irqsave(shost->host_lock, flags);
+        list_del(&rport->peers);
+        spin_unlock_irqrestore(shost->host_lock, flags);
+
+        /* Now safe to do final deletion synchronously */
+        fc_rport_final_delete(&rport->rport_delete_work);
+    }
+
+    list_for_each_entry_safe(rport, next_rport,
+            &fc_host->rport_bindings, peers) {
+        /* Cancel ALL timers and work before removing from list */
+        cancel_delayed_work_sync(&rport->fail_io_work);
+        cancel_delayed_work_sync(&rport->dev_loss_work);
+        cancel_work_sync(&rport->scan_work);
+        cancel_work_sync(&rport->stgt_delete_work);
+
+        spin_lock_irqsave(shost->host_lock, flags);
+        list_del(&rport->peers);
+        spin_unlock_irqrestore(shost->host_lock, flags);
+
+        /* Now safe to do final deletion synchronously */
+        fc_rport_final_delete(&rport->rport_delete_work);
+    }
+

     /* flush all scan work items */
     scsi_flush_work(shost);

drivers/scsi/scsi_transport_fc.c +15 -13
@@ -2983,21 +3024,22 @@ fc_rport_final_delete(struct work_struct *work)
          scsi_flush_work(shost);
 
      /*
-     * Cancel any outstanding timers. These should really exist
-     * only when rmmod'ing the LLDD and we're asking for
-     * immediate termination of the rports
+     * Cancel any outstanding delayed work synchronously.
+     * This must be done BEFORE taking spinlock and BEFORE
+     * any state changes, as cancel_delayed_work_sync() can sleep.
+     *
+     * These timers should only exist when rmmod'ing the LLDD
+     * and we're asking for immediate termination of rports.
+     */
+    cancel_delayed_work_sync(&rport->fail_io_work);
+    cancel_delayed_work_sync(&rport->dev_loss_work);
+    cancel_work_sync(&rport->scan_work);
+    /*
+     * Now safe to clear the flag under spinlock since all
+     * async work has been cancelled.
       */
      spin_lock_irqsave(shost->host_lock, flags);
-    if (rport->flags & FC_RPORT_DEVLOSS_PENDING) {
-        spin_unlock_irqrestore(shost->host_lock, flags);
-        if (!cancel_delayed_work(&rport->fail_io_work))
-            fc_flush_devloss(shost, rport);
-        if (!cancel_delayed_work(&rport->dev_loss_work))
-            fc_flush_devloss(shost, rport);
-        cancel_work_sync(&rport->scan_work);
-        spin_lock_irqsave(shost->host_lock, flags);
-        rport->flags &= ~FC_RPORT_DEVLOSS_PENDING;
-    }
+    rport->flags &= ~FC_RPORT_DEVLOSS_PENDING;
      spin_unlock_irqrestore(shost->host_lock, flags);
 
      /* Delete SCSI target and sdevs */
@@ -3027,6 +3069,7 @@ fc_rport_final_delete(struct work_struct *work)
      if (rport->devloss_work_q) {
          work_q = rport->devloss_work_q;
          rport->devloss_work_q = NULL;
+        flush_workqueue(work_q);
          destroy_workqueue(work_q);
      }