Skip to contentSkip to conversation

scsi: bnx2fc: make bnx2fc_recv_frame mp safe

PATCH Applied John Meneghini 3 participants 7 messages Jan 24, 2022 – Feb 1, 2022

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 scsi: bnx2fc: make bnx2fc_recv_frame mp safe

    Running tests with a debug kernel shows that bnx2fc_recv_frame is
    modifying the per_cpu lport stats counters in a non-mpsafe way.
    Just boot a debug kernel and run the bnx2fc driver with the hardware
    enabled.

    [ 1391.699147] BUG: using smp_processor_id() in preemptible [00000000] code: bnx2fc_
    [ 1391.699160] caller is bnx2fc_recv_frame+0xbf9/0x1760 [bnx2fc]
    [ 1391.699174] CPU: 2 PID: 4355 Comm: bnx2fc_l2_threa Kdump: loaded Tainted: G    B
    [ 1391.699180] Hardware name: HP ProLiant DL120 G7, BIOS J01 07/01/2013
    [ 1391.699183] Call Trace:
    [ 1391.699188]  dump_stack_lvl+0x57/0x7d
    [ 1391.699198]  check_preemption_disabled+0xc8/0xd0
    [ 1391.699205]  bnx2fc_recv_frame+0xbf9/0x1760 [bnx2fc]
    [ 1391.699215]  ? do_raw_spin_trylock+0xb5/0x180
    [ 1391.699221]  ? bnx2fc_npiv_create_vports.isra.0+0x4e0/0x4e0 [bnx2fc]
    [ 1391.699229]  ? bnx2fc_l2_rcv_thread+0xb7/0x3a0 [bnx2fc]
    [ 1391.699240]  bnx2fc_l2_rcv_thread+0x1af/0x3a0 [bnx2fc]
    [ 1391.699250]  ? bnx2fc_ulp_init+0xc0/0xc0 [bnx2fc]
    [ 1391.699258]  kthread+0x364/0x420
    [ 1391.699263]  ? _raw_spin_unlock_irq+0x24/0x50
    [ 1391.699268]  ? set_kthread_struct+0x100/0x100
    [ 1391.699273]  ret_from_fork+0x22/0x30

    To fix the problem: restore the old get_cpu/put_cpu code with some
    modifications to reduce the size of the critical section.
Fixes d576a5e80cd0 ("bnx2fc: Improve stats update mechanism")
---
 drivers/scsi/bnx2fc/bnx2fc_fcoe.c | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)
drivers/scsi/bnx2fc/bnx2fc_fcoe.c +13 -8
index 71fa62bd3083..e41a94dc2d1f 100644
@@ -508,7 +508,8 @@ static int bnx2fc_l2_rcv_thread(void *arg)
 
 static void bnx2fc_recv_frame(struct sk_buff *skb)
 {
-	u32 fr_len;
+	u64 crc_err;
+	u32 fr_len, fr_crc;
 	struct fc_lport *lport;
 	struct fcoe_rcv_info *fr;
 	struct fc_stats *stats;
@@ -542,6 +543,11 @@ static void bnx2fc_recv_frame(struct sk_buff *skb)
 	skb_pull(skb, sizeof(struct fcoe_hdr));
 	fr_len = skb->len - sizeof(struct fcoe_crc_eof);
 
+	stats = per_cpu_ptr(lport->stats, get_cpu());
+	stats->RxFrames++;
+	stats->RxWords += fr_len / FCOE_WORD_TO_BYTE;
+	put_cpu();
+
 	fp = (struct fc_frame *)skb;
 	fc_frame_init(fp);
 	fr_dev(fp) = lport;
@@ -624,16 +630,15 @@ static void bnx2fc_recv_frame(struct sk_buff *skb)
 		return;
 	}
 
-	stats = per_cpu_ptr(lport->stats, smp_processor_id());
-	stats->RxFrames++;
-	stats->RxWords += fr_len / FCOE_WORD_TO_BYTE;
+	fr_crc = le32_to_cpu(fr_crc(fp));
 
-	if (le32_to_cpu(fr_crc(fp)) !=
-			~crc32(~0, skb->data, fr_len)) {
-		if (stats->InvalidCRCCount < 5)
+	if (unlikely(fr_crc != ~crc32(~0, skb->data, fr_len))) {
+		stats = per_cpu_ptr(lport->stats, get_cpu());
+		crc_err = (stats->InvalidCRCCount++);
+		put_cpu();
+		if (crc_err < 5)
 			printk(KERN_WARNING PFX "dropping frame with "
 			       "CRC error\n");
-		stats->InvalidCRCCount++;
 		kfree_skb(skb);
 		return;
 	}
Signature · 1 line
2.27.0
#1 lore raw

Can I please get a review and approval for this patch?

I'd like this to get into v5.17-rc2

Thanks,

/John

Quoted 82 lines · John Meneghini
     Running tests with a debug kernel shows that bnx2fc_recv_frame is
     modifying the per_cpu lport stats counters in a non-mpsafe way.
     Just boot a debug kernel and run the bnx2fc driver with the hardware
     enabled.

     [ 1391.699147] BUG: using smp_processor_id() in preemptible [00000000] code: bnx2fc_
     [ 1391.699160] caller is bnx2fc_recv_frame+0xbf9/0x1760 [bnx2fc]
     [ 1391.699174] CPU: 2 PID: 4355 Comm: bnx2fc_l2_threa Kdump: loaded Tainted: G    B
     [ 1391.699180] Hardware name: HP ProLiant DL120 G7, BIOS J01 07/01/2013
     [ 1391.699183] Call Trace:
     [ 1391.699188]  dump_stack_lvl+0x57/0x7d
     [ 1391.699198]  check_preemption_disabled+0xc8/0xd0
     [ 1391.699205]  bnx2fc_recv_frame+0xbf9/0x1760 [bnx2fc]
     [ 1391.699215]  ? do_raw_spin_trylock+0xb5/0x180
     [ 1391.699221]  ? bnx2fc_npiv_create_vports.isra.0+0x4e0/0x4e0 [bnx2fc]
     [ 1391.699229]  ? bnx2fc_l2_rcv_thread+0xb7/0x3a0 [bnx2fc]
     [ 1391.699240]  bnx2fc_l2_rcv_thread+0x1af/0x3a0 [bnx2fc]
     [ 1391.699250]  ? bnx2fc_ulp_init+0xc0/0xc0 [bnx2fc]
     [ 1391.699258]  kthread+0x364/0x420
     [ 1391.699263]  ? _raw_spin_unlock_irq+0x24/0x50
     [ 1391.699268]  ? set_kthread_struct+0x100/0x100
     [ 1391.699273]  ret_from_fork+0x22/0x30

     To fix the problem: restore the old get_cpu/put_cpu code with some
     modifications to reduce the size of the critical section.
Fixes d576a5e80cd0 ("bnx2fc: Improve stats update mechanism")
---
  drivers/scsi/bnx2fc/bnx2fc_fcoe.c | 21 +++++++++++++--------
  1 file changed, 13 insertions(+), 8 deletions(-)
drivers/scsi/bnx2fc/bnx2fc_fcoe.c +13 -8
index 71fa62bd3083..e41a94dc2d1f 100644
@@ -508,7 +508,8 @@ static int bnx2fc_l2_rcv_thread(void *arg)
 
  static void bnx2fc_recv_frame(struct sk_buff *skb)
  {
-	u32 fr_len;
+	u64 crc_err;
+	u32 fr_len, fr_crc;
  	struct fc_lport *lport;
  	struct fcoe_rcv_info *fr;
  	struct fc_stats *stats;
@@ -542,6 +543,11 @@ static void bnx2fc_recv_frame(struct sk_buff *skb)
  	skb_pull(skb, sizeof(struct fcoe_hdr));
  	fr_len = skb->len - sizeof(struct fcoe_crc_eof);
 
+	stats = per_cpu_ptr(lport->stats, get_cpu());
+	stats->RxFrames++;
+	stats->RxWords += fr_len / FCOE_WORD_TO_BYTE;
+	put_cpu();
+
  	fp = (struct fc_frame *)skb;
  	fc_frame_init(fp);
  	fr_dev(fp) = lport;
@@ -624,16 +630,15 @@ static void bnx2fc_recv_frame(struct sk_buff *skb)
  		return;
  	}
 
-	stats = per_cpu_ptr(lport->stats, smp_processor_id());
-	stats->RxFrames++;
-	stats->RxWords += fr_len / FCOE_WORD_TO_BYTE;
+	fr_crc = le32_to_cpu(fr_crc(fp));
 
-	if (le32_to_cpu(fr_crc(fp)) !=
-			~crc32(~0, skb->data, fr_len)) {
-		if (stats->InvalidCRCCount < 5)
+	if (unlikely(fr_crc != ~crc32(~0, skb->data, fr_len))) {
+		stats = per_cpu_ptr(lport->stats, get_cpu());
+		crc_err = (stats->InvalidCRCCount++);
+		put_cpu();
+		if (crc_err < 5)
  			printk(KERN_WARNING PFX "dropping frame with "
  			       "CRC error\n");
-		stats->InvalidCRCCount++;
  		kfree_skb(skb);
  		return;
  	}
maintainer Acked-by on v1 1/1
#2 lore raw
In reply to John Meneghini

Hi John,

Quoted 89 lines

-----Original Message-----
From: John Meneghini <[email protected]>
Sent: Monday, January 24, 2022 8:21 PM
To: Saurav Kashyap <[email protected]>
Cc: Nilesh Javali <[email protected]>; [email protected]; linux- [email protected]; GR-QLogic-Storage-Upstream <GR-QLogic-Storage- [email protected]>; [email protected]; [email protected]
Subject: [EXT] [PATCH] scsi: bnx2fc: make bnx2fc_recv_frame mp safe

External Email

----------------------------------------------------------------------
Running tests with a debug kernel shows that bnx2fc_recv_frame is modifying the per_cpu lport stats counters in a non-mpsafe way. Just boot a debug kernel and run the bnx2fc driver with the hardware enabled.

[ 1391.699147] BUG: using smp_processor_id() in preemptible [00000000]
code: bnx2fc_

    [ 1391.699160] caller is bnx2fc_recv_frame+0xbf9/0x1760 [bnx2fc]
    [ 1391.699174] CPU: 2 PID: 4355 Comm: bnx2fc_l2_threa Kdump: loaded
Tainted: G    B
    [ 1391.699180] Hardware name: HP ProLiant DL120 G7, BIOS J01
07/01/2013
    [ 1391.699183] Call Trace:
    [ 1391.699188]  dump_stack_lvl+0x57/0x7d
    [ 1391.699198]  check_preemption_disabled+0xc8/0xd0
    [ 1391.699205]  bnx2fc_recv_frame+0xbf9/0x1760 [bnx2fc]
    [ 1391.699215]  ? do_raw_spin_trylock+0xb5/0x180
    [ 1391.699221]  ? bnx2fc_npiv_create_vports.isra.0+0x4e0/0x4e0 [bnx2fc]
    [ 1391.699229]  ? bnx2fc_l2_rcv_thread+0xb7/0x3a0 [bnx2fc]
    [ 1391.699240]  bnx2fc_l2_rcv_thread+0x1af/0x3a0 [bnx2fc]
    [ 1391.699250]  ? bnx2fc_ulp_init+0xc0/0xc0 [bnx2fc]
    [ 1391.699258]  kthread+0x364/0x420
    [ 1391.699263]  ? _raw_spin_unlock_irq+0x24/0x50
    [ 1391.699268]  ? set_kthread_struct+0x100/0x100
    [ 1391.699273]  ret_from_fork+0x22/0x30

To fix the problem: restore the old get_cpu/put_cpu code with some modifications to reduce the size of the critical section.

Fixes d576a5e80cd0 ("bnx2fc: Improve stats update mechanism")
---
 drivers/scsi/bnx2fc/bnx2fc_fcoe.c | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)
drivers/scsi/bnx2fc/bnx2fc_fcoe.c +10 -7
index 71fa62bd3083..e41a94dc2d1f 100644
@@ -508,7 +508,8 @@ static int bnx2fc_l2_rcv_thread(void *arg)
 
 static void bnx2fc_recv_frame(struct sk_buff *skb)
 {
-	u32 fr_len;
+	u64 crc_err;
+	u32 fr_len, fr_crc;
 	struct fc_lport *lport;
 	struct fcoe_rcv_info *fr;
 	struct fc_stats *stats;
@@ -542,6 +543,11 @@ static void bnx2fc_recv_frame(struct sk_buff *skb)
 	skb_pull(skb, sizeof(struct fcoe_hdr));
 	fr_len = skb->len - sizeof(struct fcoe_crc_eof);
 
+	stats = per_cpu_ptr(lport->stats, get_cpu());
+	stats->RxFrames++;
+	stats->RxWords += fr_len / FCOE_WORD_TO_BYTE;
+	put_cpu();
+
 	fp = (struct fc_frame *)skb;
 	fc_frame_init(fp);
 	fr_dev(fp) = lport;
@@ -624,16 +630,15 @@ static void bnx2fc_recv_frame(struct sk_buff *skb)
 		return;
 	}
 
-	stats = per_cpu_ptr(lport->stats, smp_processor_id());
-	stats->RxFrames++;
-	stats->RxWords += fr_len / FCOE_WORD_TO_BYTE;
+	fr_crc = le32_to_cpu(fr_crc(fp));
 
-	if (le32_to_cpu(fr_crc(fp)) !=
-			~crc32(~0, skb->data, fr_len)) {
-		if (stats->InvalidCRCCount < 5)
+	if (unlikely(fr_crc != ~crc32(~0, skb->data, fr_len))) {
+		stats = per_cpu_ptr(lport->stats, get_cpu());

drivers/scsi/bnx2fc/bnx2fc_fcoe.c static void bnx2fc_recv_frame(struct sk_buff *skb)

Quoted 10 lines
drivers/scsi/bnx2fc/bnx2fc_fcoe.c +3 -2
+		crc_err = (stats->InvalidCRCCount++);
+		put_cpu();
+		if (crc_err < 5)
 			printk(KERN_WARNING PFX "dropping frame with "
 			       "CRC error\n");
-		stats->InvalidCRCCount++;
 		kfree_skb(skb);
 		return;
 	}
--

Thanks for the patch.

Quoted 1 line

2.27.0

#3 lore raw

Thanks.

Martin, is it too late to get this into staging for v5.17-rc2?

Please merge this patch.

/John

Quoted 108 lines · Saurav Kashyap

Hi John,

Quoted 89 lines

-----Original Message-----
From: John Meneghini <[email protected]>
Sent: Monday, January 24, 2022 8:21 PM
To: Saurav Kashyap <[email protected]>
Cc: Nilesh Javali <[email protected]>; [email protected]; linux- [email protected]; GR-QLogic-Storage-Upstream <GR-QLogic-Storage- [email protected]>; [email protected]; [email protected]
Subject: [EXT] [PATCH] scsi: bnx2fc: make bnx2fc_recv_frame mp safe

External Email

----------------------------------------------------------------------
Running tests with a debug kernel shows that bnx2fc_recv_frame is modifying the per_cpu lport stats counters in a non-mpsafe way. Just boot a debug kernel and run the bnx2fc driver with the hardware enabled.

[ 1391.699147] BUG: using smp_processor_id() in preemptible [00000000]
code: bnx2fc_

     [ 1391.699160] caller is bnx2fc_recv_frame+0xbf9/0x1760 [bnx2fc]
     [ 1391.699174] CPU: 2 PID: 4355 Comm: bnx2fc_l2_threa Kdump: loaded
Tainted: G    B
     [ 1391.699180] Hardware name: HP ProLiant DL120 G7, BIOS J01
07/01/2013
     [ 1391.699183] Call Trace:
     [ 1391.699188]  dump_stack_lvl+0x57/0x7d
     [ 1391.699198]  check_preemption_disabled+0xc8/0xd0
     [ 1391.699205]  bnx2fc_recv_frame+0xbf9/0x1760 [bnx2fc]
     [ 1391.699215]  ? do_raw_spin_trylock+0xb5/0x180
     [ 1391.699221]  ? bnx2fc_npiv_create_vports.isra.0+0x4e0/0x4e0 [bnx2fc]
     [ 1391.699229]  ? bnx2fc_l2_rcv_thread+0xb7/0x3a0 [bnx2fc]
     [ 1391.699240]  bnx2fc_l2_rcv_thread+0x1af/0x3a0 [bnx2fc]
     [ 1391.699250]  ? bnx2fc_ulp_init+0xc0/0xc0 [bnx2fc]
     [ 1391.699258]  kthread+0x364/0x420
     [ 1391.699263]  ? _raw_spin_unlock_irq+0x24/0x50
     [ 1391.699268]  ? set_kthread_struct+0x100/0x100
     [ 1391.699273]  ret_from_fork+0x22/0x30

To fix the problem: restore the old get_cpu/put_cpu code with some modifications to reduce the size of the critical section.

Fixes d576a5e80cd0 ("bnx2fc: Improve stats update mechanism")
---
  drivers/scsi/bnx2fc/bnx2fc_fcoe.c | 21 +++++++++++++--------
  1 file changed, 13 insertions(+), 8 deletions(-)
drivers/scsi/bnx2fc/bnx2fc_fcoe.c +10 -7
index 71fa62bd3083..e41a94dc2d1f 100644
@@ -508,7 +508,8 @@ static int bnx2fc_l2_rcv_thread(void *arg)
 
  static void bnx2fc_recv_frame(struct sk_buff *skb)
  {
-	u32 fr_len;
+	u64 crc_err;
+	u32 fr_len, fr_crc;
  	struct fc_lport *lport;
  	struct fcoe_rcv_info *fr;
  	struct fc_stats *stats;
@@ -542,6 +543,11 @@ static void bnx2fc_recv_frame(struct sk_buff *skb)
  	skb_pull(skb, sizeof(struct fcoe_hdr));
  	fr_len = skb->len - sizeof(struct fcoe_crc_eof);
 
+	stats = per_cpu_ptr(lport->stats, get_cpu());
+	stats->RxFrames++;
+	stats->RxWords += fr_len / FCOE_WORD_TO_BYTE;
+	put_cpu();
+
  	fp = (struct fc_frame *)skb;
  	fc_frame_init(fp);
  	fr_dev(fp) = lport;
@@ -624,16 +630,15 @@ static void bnx2fc_recv_frame(struct sk_buff *skb)
  		return;
  	}
 
-	stats = per_cpu_ptr(lport->stats, smp_processor_id());
-	stats->RxFrames++;
-	stats->RxWords += fr_len / FCOE_WORD_TO_BYTE;
+	fr_crc = le32_to_cpu(fr_crc(fp));
 
-	if (le32_to_cpu(fr_crc(fp)) !=
-			~crc32(~0, skb->data, fr_len)) {
-		if (stats->InvalidCRCCount < 5)
+	if (unlikely(fr_crc != ~crc32(~0, skb->data, fr_len))) {
+		stats = per_cpu_ptr(lport->stats, get_cpu());

drivers/scsi/bnx2fc/bnx2fc_fcoe.c static void bnx2fc_recv_frame(struct sk_buff *skb)

Quoted 10 lines
drivers/scsi/bnx2fc/bnx2fc_fcoe.c +3 -2
+		crc_err = (stats->InvalidCRCCount++);
+		put_cpu();
+		if (crc_err < 5)
  			printk(KERN_WARNING PFX "dropping frame with "
  			       "CRC error\n");
-		stats->InvalidCRCCount++;
  		kfree_skb(skb);
  		return;
  	}
--

Thanks for the patch.

Quoted 1 line

2.27.0

#4 lore raw

John,

Erm. Hit the wrong key. Applied to 5.17/scsi-fixes, of course.

Quoted 1 line

Martin, is it too late to get this into staging for v5.17-rc2?

Patches sit in linux-next for about a week before they get sent to Linus.

Signature · 1 line
Martin K. Petersen	Oracle Linux Engineering
#5 lore raw
In reply to John Meneghini

John,

Applied to 5.18/scsi-staging, thanks!

Please make sure your commit descriptions are left aligned:

Quoted 4 lines

Running tests with a debug kernel shows that bnx2fc_recv_frame is modifying the per_cpu lport stats counters in a non-mpsafe way. Just boot a debug kernel and run the bnx2fc driver with the hardware enabled.

^^^^^

Signature · 1 line
Martin K. Petersen	Oracle Linux Engineering
#6 lore raw
In reply to John Meneghini
Quoted 23 lines · John Meneghini
    Running tests with a debug kernel shows that bnx2fc_recv_frame is
    modifying the per_cpu lport stats counters in a non-mpsafe way.
    Just boot a debug kernel and run the bnx2fc driver with the hardware
    enabled.

    [ 1391.699147] BUG: using smp_processor_id() in preemptible [00000000] code: bnx2fc_
    [ 1391.699160] caller is bnx2fc_recv_frame+0xbf9/0x1760 [bnx2fc]
    [ 1391.699174] CPU: 2 PID: 4355 Comm: bnx2fc_l2_threa Kdump: loaded Tainted: G    B
    [ 1391.699180] Hardware name: HP ProLiant DL120 G7, BIOS J01 07/01/2013
    [ 1391.699183] Call Trace:
    [ 1391.699188]  dump_stack_lvl+0x57/0x7d
    [ 1391.699198]  check_preemption_disabled+0xc8/0xd0
    [ 1391.699205]  bnx2fc_recv_frame+0xbf9/0x1760 [bnx2fc]
    [ 1391.699215]  ? do_raw_spin_trylock+0xb5/0x180
    [ 1391.699221]  ? bnx2fc_npiv_create_vports.isra.0+0x4e0/0x4e0 [bnx2fc]
    [ 1391.699229]  ? bnx2fc_l2_rcv_thread+0xb7/0x3a0 [bnx2fc]
    [ 1391.699240]  bnx2fc_l2_rcv_thread+0x1af/0x3a0 [bnx2fc]
    [ 1391.699250]  ? bnx2fc_ulp_init+0xc0/0xc0 [bnx2fc]
    [ 1391.699258]  kthread+0x364/0x420
    [ 1391.699263]  ? _raw_spin_unlock_irq+0x24/0x50
    [ 1391.699268]  ? set_kthread_struct+0x100/0x100
    [ 1391.699273]  ret_from_fork+0x22/0x30
Quoted 1 line

[...]

Applied to 5.17/scsi-fixes, thanks!

[1/1] scsi: bnx2fc: make bnx2fc_recv_frame mp safe

      https://git.kernel.org/mkp/scsi/c/936bd03405fc
Signature · 1 line
Martin K. Petersen	Oracle Linux Engineering