Skip to contentSkip to conversation

genirq: change force_irqthreads from bool test to static key

PATCH Applied Tanner Love 3 participants 4 messages Jun 2, 2021 – Aug 10, 2021

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 genirq: change force_irqthreads from bool test to static key

From: Tanner Love <[email protected]>

With CONFIG_IRQ_FORCED_THREADING=y, testing the bool force_irqthreads could incur a cache line miss in invoke_softirq().

Replace the test with a static key to avoid the potential cache miss.

---
 drivers/ide/ide-iops.c    | 7 +++----
 include/linux/interrupt.h | 4 +++-
 kernel/irq/manage.c       | 6 +++---
 3 files changed, 9 insertions(+), 8 deletions(-)
drivers/ide/ide-iops.c +3 -4
index f2be127ee96e..f86cdb8451e6 100644
@@ -109,7 +109,6 @@ int __ide_wait_stat(ide_drive_t *drive, u8 good, u8 bad,
 	ide_hwif_t *hwif = drive->hwif;
 	const struct ide_tp_ops *tp_ops = hwif->tp_ops;
 	unsigned long flags;
-	bool irqs_threaded = force_irqthreads;
 	int i;
 	u8 stat;
 
@@ -117,7 +116,7 @@ int __ide_wait_stat(ide_drive_t *drive, u8 good, u8 bad,
 	stat = tp_ops->read_status(hwif);
 
 	if (stat & ATA_BUSY) {
-		if (!irqs_threaded) {
+		if (!force_irqthreads) {
 			local_save_flags(flags);
 			local_irq_enable_in_hardirq();
 		}
@@ -133,13 +132,13 @@ int __ide_wait_stat(ide_drive_t *drive, u8 good, u8 bad,
 				if ((stat & ATA_BUSY) == 0)
 					break;
 
-				if (!irqs_threaded)
+				if (!force_irqthreads)
 					local_irq_restore(flags);
 				*rstat = stat;
 				return -EBUSY;
 			}
 		}
-		if (!irqs_threaded)
+		if (!force_irqthreads)
 			local_irq_restore(flags);
 	}
 	/*
include/linux/interrupt.h +3 -1
index 4777850a6dc7..9e676c351f23 100644
@@ -13,6 +13,7 @@
 #include <linux/hrtimer.h>
 #include <linux/kref.h>
 #include <linux/workqueue.h>
+#include <linux/jump_label.h>
 
 #include <linux/atomic.h>
 #include <asm/ptrace.h>
@@ -504,7 +505,8 @@ extern int irq_set_irqchip_state(unsigned int irq, enum irqchip_irq_state which,
 # ifdef CONFIG_PREEMPT_RT
 #  define force_irqthreads	(true)
 # else
-extern bool force_irqthreads;
+DECLARE_STATIC_KEY_FALSE(force_irqthreads_key);
+#  define force_irqthreads	(static_branch_unlikely(&force_irqthreads_key))
 # endif
 #else
 #define force_irqthreads	(0)
kernel/irq/manage.c +3 -3
index 4c14356543d9..395945e54929 100644
@@ -25,12 +25,12 @@
 #include "internals.h"
 
 #if defined(CONFIG_IRQ_FORCED_THREADING) && !defined(CONFIG_PREEMPT_RT)
-__read_mostly bool force_irqthreads;
-EXPORT_SYMBOL_GPL(force_irqthreads);
+DEFINE_STATIC_KEY_FALSE(force_irqthreads_key);
+EXPORT_SYMBOL_GPL(force_irqthreads_key);
 
 static int __init setup_forced_irqthreads(char *arg)
 {
-	force_irqthreads = true;
+	static_branch_enable(&force_irqthreads_key);
 	return 0;
 }
 early_param("threadirqs", setup_forced_irqthreads);
Signature · 1 line
2.32.0.rc0.204.g9fa02ecfa5-goog
Reviewed-by on v1 1/1
#1 lore raw
Quoted 66 lines · Tanner Love

From: Tanner Love <[email protected]>

With CONFIG_IRQ_FORCED_THREADING=y, testing the bool force_irqthreads could incur a cache line miss in invoke_softirq().

Replace the test with a static key to avoid the potential cache miss.

---
 drivers/ide/ide-iops.c    | 7 +++----
 include/linux/interrupt.h | 4 +++-
 kernel/irq/manage.c       | 6 +++---
 3 files changed, 9 insertions(+), 8 deletions(-)
drivers/ide/ide-iops.c +3 -4
index f2be127ee96e..f86cdb8451e6 100644
@@ -109,7 +109,6 @@ int __ide_wait_stat(ide_drive_t *drive, u8 good, u8 bad,
 	ide_hwif_t *hwif = drive->hwif;
 	const struct ide_tp_ops *tp_ops = hwif->tp_ops;
 	unsigned long flags;
-	bool irqs_threaded = force_irqthreads;
 	int i;
 	u8 stat;
 
@@ -117,7 +116,7 @@ int __ide_wait_stat(ide_drive_t *drive, u8 good, u8 bad,
 	stat = tp_ops->read_status(hwif);
 
 	if (stat & ATA_BUSY) {
-		if (!irqs_threaded) {
+		if (!force_irqthreads) {
 			local_save_flags(flags);
 			local_irq_enable_in_hardirq();
 		}
@@ -133,13 +132,13 @@ int __ide_wait_stat(ide_drive_t *drive, u8 good, u8 bad,
 				if ((stat & ATA_BUSY) == 0)
 					break;
 
-				if (!irqs_threaded)
+				if (!force_irqthreads)
 					local_irq_restore(flags);
 				*rstat = stat;
 				return -EBUSY;
 			}
 		}
-		if (!irqs_threaded)
+		if (!force_irqthreads)
 			local_irq_restore(flags);
 	}
 	/*
include/linux/interrupt.h +1 -0
index 4777850a6dc7..9e676c351f23 100644
@@ -13,6 +13,7 @@
 #include <linux/hrtimer.h>
 #include <linux/kref.h>
 #include <linux/workqueue.h>
+#include <linux/jump_label.h>
 
 #include <linux/atomic.h>
 #include <asm/ptrace.h>

include/linux/interrupt.h extern int irq_set_irqchip_state(unsigned int irq, enum irqchip_irq_state which, · line 509

Quoted 6 lines
include/linux/interrupt.h +2 -1
@@ -504,7 +505,8 @@ extern int irq_set_irqchip_state(unsigned int irq, enum irqchip_irq_state which,
 # ifdef CONFIG_PREEMPT_RT
 #  define force_irqthreads	(true)
 # else
-extern bool force_irqthreads;
+DECLARE_STATIC_KEY_FALSE(force_irqthreads_key);
+#  define force_irqthreads	(static_branch_unlikely(&force_irqthreads_key))

I like this idiom, though it did make me look twice because I thought you were missing the static_branch...() calls above. ;)

-Kees

Quoted 26 lines
include/linux/interrupt.h +0 -0
 # endif
 #else
 #define force_irqthreads	(0)
kernel/irq/manage.c +3 -3
index 4c14356543d9..395945e54929 100644
@@ -25,12 +25,12 @@
 #include "internals.h"
 
 #if defined(CONFIG_IRQ_FORCED_THREADING) && !defined(CONFIG_PREEMPT_RT)
-__read_mostly bool force_irqthreads;
-EXPORT_SYMBOL_GPL(force_irqthreads);
+DEFINE_STATIC_KEY_FALSE(force_irqthreads_key);
+EXPORT_SYMBOL_GPL(force_irqthreads_key);
 
 static int __init setup_forced_irqthreads(char *arg)
 {
-	force_irqthreads = true;
+	static_branch_enable(&force_irqthreads_key);
 	return 0;
 }
 early_param("threadirqs", setup_forced_irqthreads);
Signature · 1 line
2.32.0.rc0.204.g9fa02ecfa5-goog
Signature · 1 line
Kees Cook
merge bot Applied
#2 lore raw

[tip: irq/core] genirq: Change force_irqthreads to a static key

In reply to Tanner Love
The following commit has been merged into the irq/core branch of tip:

The following commit has been merged into the irq/core branch of tip:

Commit-ID:     af5b7fe6bb77ac775d446e2f25f013d5df551e9a
Gitweb:        https://git.kernel.org/tip/af5b7fe6bb77ac775d446e2f25f013d5df551e9a
Author:        Tanner Love <[email protected]>
AuthorDate:    Wed, 02 Jun 2021 14:03:38 -04:00
Committer:     Thomas Gleixner <[email protected]>
CommitterDate: Tue, 10 Aug 2021 16:06:55 +02:00

genirq: Change force_irqthreads to a static key

With CONFIG_IRQ_FORCED_THREADING=y, testing the boolean force_irqthreads could incur a cache line miss in invoke_softirq() and other places.

Replace the test with a static key to avoid the potential cache miss.

[ tglx: Dropped the IDE part, removed the export and updated blk-mq ]

---
 block/blk-mq.c            |  2 +-
 include/linux/interrupt.h |  8 +++++---
 kernel/irq/manage.c       |  9 ++++-----
 kernel/softirq.c          |  2 +-
 4 files changed, 11 insertions(+), 10 deletions(-)
block/blk-mq.c +1 -1
index 2c4ac51..572d8ab 100644
@@ -606,7 +606,7 @@ static inline bool blk_mq_complete_need_ipi(struct request *rq)
 	 * This is probably worse than completing the request on a different
 	 * cache domain.
 	 */
-	if (force_irqthreads)
+	if (force_irqthreads())
 		return false;
 
 	/* same CPU or cache domain?  Complete locally */
include/linux/interrupt.h +5 -3
index 2ed65b0..1f22a30 100644
@@ -13,6 +13,7 @@
 #include <linux/hrtimer.h>
 #include <linux/kref.h>
 #include <linux/workqueue.h>
+#include <linux/jump_label.h>
 
 #include <linux/atomic.h>
 #include <asm/ptrace.h>
@@ -474,12 +475,13 @@ extern int irq_set_irqchip_state(unsigned int irq, enum irqchip_irq_state which,
 
 #ifdef CONFIG_IRQ_FORCED_THREADING
 # ifdef CONFIG_PREEMPT_RT
-#  define force_irqthreads	(true)
+#  define force_irqthreads()	(true)
 # else
-extern bool force_irqthreads;
+DECLARE_STATIC_KEY_FALSE(force_irqthreads_key);
+#  define force_irqthreads()	(static_branch_unlikely(&force_irqthreads_key))
 # endif
 #else
-#define force_irqthreads	(0)
+#define force_irqthreads()	(false)
 #endif
 
 #ifndef local_softirq_pending
kernel/irq/manage.c +4 -5
index 766468a..bb3c51d 100644
@@ -25,12 +25,11 @@
 #include "internals.h"
 
 #if defined(CONFIG_IRQ_FORCED_THREADING) && !defined(CONFIG_PREEMPT_RT)
-__read_mostly bool force_irqthreads;
-EXPORT_SYMBOL_GPL(force_irqthreads);
+DEFINE_STATIC_KEY_FALSE(force_irqthreads_key);
 
 static int __init setup_forced_irqthreads(char *arg)
 {
-	force_irqthreads = true;
+	static_branch_enable(&force_irqthreads_key);
 	return 0;
 }
 early_param("threadirqs", setup_forced_irqthreads);
@@ -1260,8 +1259,8 @@ static int irq_thread(void *data)
 	irqreturn_t (*handler_fn)(struct irq_desc *desc,
 			struct irqaction *action);
 
-	if (force_irqthreads && test_bit(IRQTF_FORCED_THREAD,
-					&action->thread_flags))
+	if (force_irqthreads() && test_bit(IRQTF_FORCED_THREAD,
+					   &action->thread_flags))
 		handler_fn = irq_forced_thread_fn;
 	else
 		handler_fn = irq_thread_fn;
kernel/softirq.c +1 -1
index f3a0121..322b65d 100644
@@ -422,7 +422,7 @@ static inline void invoke_softirq(void)
 	if (ksoftirqd_running(local_softirq_pending()))
 		return;
 
-	if (!force_irqthreads || !__this_cpu_read(ksoftirqd)) {
+	if (!force_irqthreads() || !__this_cpu_read(ksoftirqd)) {
 #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
 		/*
 		 * We can safely execute softirq on the current stack if
merge bot Applied
#3 lore raw

[tip: irq/core] genirq: Change force_irqthreads to a static key

In reply to Tanner Love
The following commit has been merged into the irq/core branch of tip:

The following commit has been merged into the irq/core branch of tip:

Commit-ID:     91cc470e797828d779cd4c1efbe8519bcb358bae
Gitweb:        https://git.kernel.org/tip/91cc470e797828d779cd4c1efbe8519bcb358bae
Author:        Tanner Love <[email protected]>
AuthorDate:    Wed, 02 Jun 2021 14:03:38 -04:00
Committer:     Thomas Gleixner <[email protected]>
CommitterDate: Tue, 10 Aug 2021 22:50:07 +02:00

genirq: Change force_irqthreads to a static key

With CONFIG_IRQ_FORCED_THREADING=y, testing the boolean force_irqthreads could incur a cache line miss in invoke_softirq() and other places.

Replace the test with a static key to avoid the potential cache miss.

[ tglx: Dropped the IDE part, removed the export and updated blk-mq ]

---
 block/blk-mq.c            |  2 +-
 include/linux/interrupt.h |  8 +++++---
 kernel/irq/manage.c       | 11 +++++------
 kernel/softirq.c          |  2 +-
 4 files changed, 12 insertions(+), 11 deletions(-)
block/blk-mq.c +1 -1
index 2c4ac51..572d8ab 100644
@@ -606,7 +606,7 @@ static inline bool blk_mq_complete_need_ipi(struct request *rq)
 	 * This is probably worse than completing the request on a different
 	 * cache domain.
 	 */
-	if (force_irqthreads)
+	if (force_irqthreads())
 		return false;
 
 	/* same CPU or cache domain?  Complete locally */
include/linux/interrupt.h +5 -3
index 2ed65b0..1f22a30 100644
@@ -13,6 +13,7 @@
 #include <linux/hrtimer.h>
 #include <linux/kref.h>
 #include <linux/workqueue.h>
+#include <linux/jump_label.h>
 
 #include <linux/atomic.h>
 #include <asm/ptrace.h>
@@ -474,12 +475,13 @@ extern int irq_set_irqchip_state(unsigned int irq, enum irqchip_irq_state which,
 
 #ifdef CONFIG_IRQ_FORCED_THREADING
 # ifdef CONFIG_PREEMPT_RT
-#  define force_irqthreads	(true)
+#  define force_irqthreads()	(true)
 # else
-extern bool force_irqthreads;
+DECLARE_STATIC_KEY_FALSE(force_irqthreads_key);
+#  define force_irqthreads()	(static_branch_unlikely(&force_irqthreads_key))
 # endif
 #else
-#define force_irqthreads	(0)
+#define force_irqthreads()	(false)
 #endif
 
 #ifndef local_softirq_pending
kernel/irq/manage.c +5 -6
index 766468a..34a66c4 100644
@@ -25,12 +25,11 @@
 #include "internals.h"
 
 #if defined(CONFIG_IRQ_FORCED_THREADING) && !defined(CONFIG_PREEMPT_RT)
-__read_mostly bool force_irqthreads;
-EXPORT_SYMBOL_GPL(force_irqthreads);
+DEFINE_STATIC_KEY_FALSE(force_irqthreads_key);
 
 static int __init setup_forced_irqthreads(char *arg)
 {
-	force_irqthreads = true;
+	static_branch_enable(&force_irqthreads_key);
 	return 0;
 }
 early_param("threadirqs", setup_forced_irqthreads);
@@ -1260,8 +1259,8 @@ static int irq_thread(void *data)
 	irqreturn_t (*handler_fn)(struct irq_desc *desc,
 			struct irqaction *action);
 
-	if (force_irqthreads && test_bit(IRQTF_FORCED_THREAD,
-					&action->thread_flags))
+	if (force_irqthreads() && test_bit(IRQTF_FORCED_THREAD,
+					   &action->thread_flags))
 		handler_fn = irq_forced_thread_fn;
 	else
 		handler_fn = irq_thread_fn;
@@ -1322,7 +1321,7 @@ EXPORT_SYMBOL_GPL(irq_wake_thread);
 
 static int irq_setup_forced_threading(struct irqaction *new)
 {
-	if (!force_irqthreads)
+	if (!force_irqthreads())
 		return 0;
 	if (new->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT))
 		return 0;
kernel/softirq.c +1 -1
index f3a0121..322b65d 100644
@@ -422,7 +422,7 @@ static inline void invoke_softirq(void)
 	if (ksoftirqd_running(local_softirq_pending()))
 		return;
 
-	if (!force_irqthreads || !__this_cpu_read(ksoftirqd)) {
+	if (!force_irqthreads() || !__this_cpu_read(ksoftirqd)) {
 #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
 		/*
 		 * We can safely execute softirq on the current stack if