Skip to contentSkip to conversation

genirq: Get rid of global lock in irq_do_set_affinity()

PATCH Applied Marc Zyngier 3 participants 4 messages Aug 26, 2024 – Aug 27, 2024 IRQ

v1 of 2 Applied Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 genirq: Get rid of global lock in irq_do_set_affinity()

Kunkun Jiang reports that for a workload involving the simultaneous startup of a large number of VMs (for a total of about 200 vcpus), a lot of CPU time gets spent on spinning on the tmp_mask_lock that exists as a static raw spinlock in irq_do_set_affinity(). This lock protects a global cpumask (tmp_mask) that is used as a temporary variable to compute the resulting affinity.

While this is triggered by KVM issuing a irq_set_affinity() call each time a vcpu is about to execute, it is obvious that having a single global resource is not very scalable, and that we could do something better.

Since a cpumask can be a fairly large structure on systems with a high core count, a stack allocation is not really appropriate. Instead, turn the global cpumask into a per-CPU variable, removing the need for locking altogether as we are not preemptible at this point.

---
 kernel/irq/manage.c | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)
kernel/irq/manage.c +10 -11
index dd53298ef1a5..b6aa259ac749 100644
@@ -224,15 +224,16 @@ int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,
 	struct irq_desc *desc = irq_data_to_desc(data);
 	struct irq_chip *chip = irq_data_get_irq_chip(data);
 	const struct cpumask  *prog_mask;
+	struct cpumask *tmp_mask;
 	int ret;
 
-	static DEFINE_RAW_SPINLOCK(tmp_mask_lock);
-	static struct cpumask tmp_mask;
+	static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
 
 	if (!chip || !chip->irq_set_affinity)
 		return -EINVAL;
 
-	raw_spin_lock(&tmp_mask_lock);
+	tmp_mask = this_cpu_ptr(&__tmp_mask);
+
 	/*
 	 * If this is a managed interrupt and housekeeping is enabled on
 	 * it check whether the requested affinity mask intersects with
@@ -258,11 +259,11 @@ int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,
 
 		hk_mask = housekeeping_cpumask(HK_TYPE_MANAGED_IRQ);
 
-		cpumask_and(&tmp_mask, mask, hk_mask);
-		if (!cpumask_intersects(&tmp_mask, cpu_online_mask))
+		cpumask_and(tmp_mask, mask, hk_mask);
+		if (!cpumask_intersects(tmp_mask, cpu_online_mask))
 			prog_mask = mask;
 		else
-			prog_mask = &tmp_mask;
+			prog_mask = tmp_mask;
 	} else {
 		prog_mask = mask;
 	}
@@ -272,16 +273,14 @@ int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,
 	 * unless we are being asked to force the affinity (in which
 	 * case we do as we are told).
 	 */
-	cpumask_and(&tmp_mask, prog_mask, cpu_online_mask);
-	if (!force && !cpumask_empty(&tmp_mask))
-		ret = chip->irq_set_affinity(data, &tmp_mask, force);
+	cpumask_and(tmp_mask, prog_mask, cpu_online_mask);
+	if (!force && !cpumask_empty(tmp_mask))
+		ret = chip->irq_set_affinity(data, tmp_mask, force);
 	else if (force)
 		ret = chip->irq_set_affinity(data, mask, force);
 	else
 		ret = -EINVAL;
 
-	raw_spin_unlock(&tmp_mask_lock);
-
 	switch (ret) {
 	case IRQ_SET_MASK_OK:
 	case IRQ_SET_MASK_OK_DONE:
Signature · 1 line
2.39.2
merge bot Applied
#1 lore raw
The following commit has been merged into the irq/core branch of tip:

The following commit has been merged into the irq/core branch of tip:

Commit-ID:     6c70d79f363c0c9a712e107b9c4d3644ca8c7490
Gitweb:        https://git.kernel.org/tip/6c70d79f363c0c9a712e107b9c4d3644ca8c7490
Author:        Marc Zyngier <[email protected]>
AuthorDate:    Mon, 26 Aug 2024 09:06:18 +01:00
Committer:     Thomas Gleixner <[email protected]>
CommitterDate: Mon, 26 Aug 2024 12:52:35 +02:00

genirq: Get rid of global lock in irq_do_set_affinity()

Kunkun Jiang reports that for a workload involving the simultaneous startup of a large number of VMs (for a total of about 200 vcpus), a lot of CPU time gets spent on spinning on the tmp_mask_lock that exists as a static raw spinlock in irq_do_set_affinity(). This lock protects a global cpumask (tmp_mask) that is used as a temporary variable to compute the resulting affinity.

While this is triggered by KVM issuing a irq_set_affinity() call each time a vcpu is about to execute, it is obvious that having a single global resource is not very scalable.

Since a cpumask can be a fairly large structure on systems with a high core count, a stack allocation is not really appropriate. Instead, turn the global cpumask into a per-CPU variable, removing the need for locking altogether as the code is executed with preemption and interrupts disabled.

---
 kernel/irq/manage.c | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)
kernel/irq/manage.c +10 -11
index dd53298..b6aa259 100644
@@ -224,15 +224,16 @@ int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,
 	struct irq_desc *desc = irq_data_to_desc(data);
 	struct irq_chip *chip = irq_data_get_irq_chip(data);
 	const struct cpumask  *prog_mask;
+	struct cpumask *tmp_mask;
 	int ret;
 
-	static DEFINE_RAW_SPINLOCK(tmp_mask_lock);
-	static struct cpumask tmp_mask;
+	static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
 
 	if (!chip || !chip->irq_set_affinity)
 		return -EINVAL;
 
-	raw_spin_lock(&tmp_mask_lock);
+	tmp_mask = this_cpu_ptr(&__tmp_mask);
+
 	/*
 	 * If this is a managed interrupt and housekeeping is enabled on
 	 * it check whether the requested affinity mask intersects with
@@ -258,11 +259,11 @@ int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,
 
 		hk_mask = housekeeping_cpumask(HK_TYPE_MANAGED_IRQ);
 
-		cpumask_and(&tmp_mask, mask, hk_mask);
-		if (!cpumask_intersects(&tmp_mask, cpu_online_mask))
+		cpumask_and(tmp_mask, mask, hk_mask);
+		if (!cpumask_intersects(tmp_mask, cpu_online_mask))
 			prog_mask = mask;
 		else
-			prog_mask = &tmp_mask;
+			prog_mask = tmp_mask;
 	} else {
 		prog_mask = mask;
 	}
@@ -272,16 +273,14 @@ int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,
 	 * unless we are being asked to force the affinity (in which
 	 * case we do as we are told).
 	 */
-	cpumask_and(&tmp_mask, prog_mask, cpu_online_mask);
-	if (!force && !cpumask_empty(&tmp_mask))
-		ret = chip->irq_set_affinity(data, &tmp_mask, force);
+	cpumask_and(tmp_mask, prog_mask, cpu_online_mask);
+	if (!force && !cpumask_empty(tmp_mask))
+		ret = chip->irq_set_affinity(data, tmp_mask, force);
 	else if (force)
 		ret = chip->irq_set_affinity(data, mask, force);
 	else
 		ret = -EINVAL;
 
-	raw_spin_unlock(&tmp_mask_lock);
-
 	switch (ret) {
 	case IRQ_SET_MASK_OK:
 	case IRQ_SET_MASK_OK_DONE:
#2 lore raw
In reply to Marc Zyngier
kernel test robot noticed the following build errors:

Hi Marc,

kernel test robot noticed the following build errors:

[auto build test ERROR on tip/irq/core]
[also build test ERROR on linus/master v6.11-rc5 next-20240826] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Marc-Zyngier/genirq-Get-rid-of-global-lock-in-irq_do_set_affinity/20240826-170538
base:   tip/irq/core
patch link:    https://lore.kernel.org/r/20240826080618.3886694-1-maz%40kernel.org
patch subject: [PATCH] genirq: Get rid of global lock in irq_do_set_affinity()
config: arc-randconfig-002-20240827 (https://download.01.org/0day-ci/archive/20240827/[email protected]/config)
compiler: arceb-elf-gcc (GCC) 13.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20240827/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All errors (new ones prefixed by >>):

In file included from include/asm-generic/percpu.h:7,

                    from ./arch/arc/include/generated/asm/percpu.h:1,
                    from include/linux/irqflags.h:19,
                    from arch/arc/include/asm/smp.h:104,
                    from arch/arc/include/asm/cmpxchg.h:13,
                    from arch/arc/include/asm/atomic.h:13,
                    from include/linux/atomic.h:7,
                    from include/asm-generic/bitops/lock.h:5,
                    from arch/arc/include/asm/bitops.h:188,
                    from include/linux/bitops.h:68,
                    from include/linux/thread_info.h:27,
                    from include/asm-generic/preempt.h:5,
                    from ./arch/arc/include/generated/asm/preempt.h:1,
                    from include/linux/preempt.h:79,
                    from include/linux/spinlock.h:56,
                    from include/linux/irq.h:14,
                    from kernel/irq/manage.c:11:
   kernel/irq/manage.c: In function 'irq_do_set_affinity':
   include/linux/percpu-defs.h:92:40: error: section attribute cannot be specified for local variables
      92 |         extern __PCPU_DUMMY_ATTRS char __pcpu_unique_##name;            \
         |                                        ^~~~~~~~~~~~~~
   include/linux/percpu-defs.h:115:9: note: in expansion of macro 'DEFINE_PER_CPU_SECTION'
     115 |         DEFINE_PER_CPU_SECTION(type, name, "")
         |         ^~~~~~~~~~~~~~~~~~~~~~
   kernel/irq/manage.c:230:16: note: in expansion of macro 'DEFINE_PER_CPU'
     230 |         static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
         |                ^~~~~~~~~~~~~~
   include/linux/percpu-defs.h:93:33: error: section attribute cannot be specified for local variables
      93 |         __PCPU_DUMMY_ATTRS char __pcpu_unique_##name;                   \
         |                                 ^~~~~~~~~~~~~~
   include/linux/percpu-defs.h:115:9: note: in expansion of macro 'DEFINE_PER_CPU_SECTION'
     115 |         DEFINE_PER_CPU_SECTION(type, name, "")
         |         ^~~~~~~~~~~~~~~~~~~~~~
   kernel/irq/manage.c:230:16: note: in expansion of macro 'DEFINE_PER_CPU'
     230 |         static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
         |                ^~~~~~~~~~~~~~
>> include/linux/percpu-defs.h:93:33: error: declaration of '__pcpu_unique___tmp_mask' with no linkage follows extern declaration
      93 |         __PCPU_DUMMY_ATTRS char __pcpu_unique_##name;                   \
         |                                 ^~~~~~~~~~~~~~
   include/linux/percpu-defs.h:115:9: note: in expansion of macro 'DEFINE_PER_CPU_SECTION'
     115 |         DEFINE_PER_CPU_SECTION(type, name, "")
         |         ^~~~~~~~~~~~~~~~~~~~~~
   kernel/irq/manage.c:230:16: note: in expansion of macro 'DEFINE_PER_CPU'
     230 |         static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
         |                ^~~~~~~~~~~~~~
   include/linux/percpu-defs.h:92:40: note: previous declaration of '__pcpu_unique___tmp_mask' with type 'char'
      92 |         extern __PCPU_DUMMY_ATTRS char __pcpu_unique_##name;            \
         |                                        ^~~~~~~~~~~~~~
   include/linux/percpu-defs.h:115:9: note: in expansion of macro 'DEFINE_PER_CPU_SECTION'
     115 |         DEFINE_PER_CPU_SECTION(type, name, "")
         |         ^~~~~~~~~~~~~~~~~~~~~~
   kernel/irq/manage.c:230:16: note: in expansion of macro 'DEFINE_PER_CPU'
     230 |         static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
         |                ^~~~~~~~~~~~~~
>> kernel/irq/manage.c:230:47: error: section attribute cannot be specified for local variables
     230 |         static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
         |                                               ^~~~~~~~~~
   include/linux/percpu-defs.h:94:51: note: in definition of macro 'DEFINE_PER_CPU_SECTION'
      94 |         extern __PCPU_ATTRS(sec) __typeof__(type) name;                 \
         |                                                   ^~~~
   kernel/irq/manage.c:230:16: note: in expansion of macro 'DEFINE_PER_CPU'
     230 |         static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
         |                ^~~~~~~~~~~~~~
>> kernel/irq/manage.c:230:47: error: section attribute cannot be specified for local variables
     230 |         static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
         |                                               ^~~~~~~~~~
   include/linux/percpu-defs.h:95:51: note: in definition of macro 'DEFINE_PER_CPU_SECTION'
      95 |         __PCPU_ATTRS(sec) __weak __typeof__(type) name
         |                                                   ^~~~
   kernel/irq/manage.c:230:16: note: in expansion of macro 'DEFINE_PER_CPU'
     230 |         static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
         |                ^~~~~~~~~~~~~~
>> kernel/irq/manage.c:230:47: error: weak declaration of '__tmp_mask' must be public
     230 |         static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
         |                                               ^~~~~~~~~~
   include/linux/percpu-defs.h:95:51: note: in definition of macro 'DEFINE_PER_CPU_SECTION'
      95 |         __PCPU_ATTRS(sec) __weak __typeof__(type) name
         |                                                   ^~~~
   kernel/irq/manage.c:230:16: note: in expansion of macro 'DEFINE_PER_CPU'
     230 |         static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
         |                ^~~~~~~~~~~~~~
>> kernel/irq/manage.c:230:47: error: declaration of '__tmp_mask' with no linkage follows extern declaration
     230 |         static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
         |                                               ^~~~~~~~~~
   include/linux/percpu-defs.h:95:51: note: in definition of macro 'DEFINE_PER_CPU_SECTION'
      95 |         __PCPU_ATTRS(sec) __weak __typeof__(type) name
         |                                                   ^~~~
   kernel/irq/manage.c:230:16: note: in expansion of macro 'DEFINE_PER_CPU'
     230 |         static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
         |                ^~~~~~~~~~~~~~
   kernel/irq/manage.c:230:47: note: previous declaration of '__tmp_mask' with type 'struct cpumask'
     230 |         static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
         |                                               ^~~~~~~~~~
   include/linux/percpu-defs.h:94:51: note: in definition of macro 'DEFINE_PER_CPU_SECTION'
      94 |         extern __PCPU_ATTRS(sec) __typeof__(type) name;                 \
         |                                                   ^~~~
   kernel/irq/manage.c:230:16: note: in expansion of macro 'DEFINE_PER_CPU'
     230 |         static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
         |                ^~~~~~~~~~~~~~

vim +230 kernel/irq/manage.c

220
221 int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,
222 bool force)
223 {
224 struct irq_desc *desc = irq_data_to_desc(data);
225 struct irq_chip *chip = irq_data_get_irq_chip(data);
226 const struct cpumask *prog_mask;
227 struct cpumask *tmp_mask;
228 int ret;
229

 > 230		static DEFINE_PER_CPU(struct cpumask, __tmp_mask);

231
232 if (!chip || !chip->irq_set_affinity)
233 return -EINVAL;
234
235 tmp_mask = this_cpu_ptr(&__tmp_mask);
236
237 /*
238 * If this is a managed interrupt and housekeeping is enabled on
239 * it check whether the requested affinity mask intersects with
240 * a housekeeping CPU. If so, then remove the isolated CPUs from
241 * the mask and just keep the housekeeping CPU(s). This prevents
242 * the affinity setter from routing the interrupt to an isolated
243 * CPU to avoid that I/O submitted from a housekeeping CPU causes
244 * interrupts on an isolated one.
245 *
246 * If the masks do not intersect or include online CPU(s) then
247 * keep the requested mask. The isolated target CPUs are only
248 * receiving interrupts when the I/O operation was submitted
249 * directly from them.
250 *
251 * If all housekeeping CPUs in the affinity mask are offline, the
252 * interrupt will be migrated by the CPU hotplug code once a
253 * housekeeping CPU which belongs to the affinity mask comes
254 * online.
255 */
256 if (irqd_affinity_is_managed(data) &&
257 housekeeping_enabled(HK_TYPE_MANAGED_IRQ)) {
258 const struct cpumask *hk_mask;
259
260 hk_mask = housekeeping_cpumask(HK_TYPE_MANAGED_IRQ);
261
262 cpumask_and(tmp_mask, mask, hk_mask);
263 if (!cpumask_intersects(tmp_mask, cpu_online_mask))
264 prog_mask = mask;
265 else
266 prog_mask = tmp_mask;
267 } else {
268 prog_mask = mask;
269 }
270
271 /*
272 * Make sure we only provide online CPUs to the irqchip,
273 * unless we are being asked to force the affinity (in which
274 * case we do as we are told).
275 */
276 cpumask_and(tmp_mask, prog_mask, cpu_online_mask);
277 if (!force && !cpumask_empty(tmp_mask))
278 ret = chip->irq_set_affinity(data, tmp_mask, force);
279 else if (force)
280 ret = chip->irq_set_affinity(data, mask, force);
281 else
282 ret = -EINVAL;
283
284 switch (ret) {
285 case IRQ_SET_MASK_OK:
286 case IRQ_SET_MASK_OK_DONE:
287 cpumask_copy(desc->irq_common_data.affinity, mask);
288 fallthrough;
289 case IRQ_SET_MASK_OK_NOCOPY:
290 irq_validate_effective_affinity(data);
291 irq_set_thread_affinity(desc);
292 ret = 0;
293 }
294
295 return ret;
296 }
297

Signature · 2 lines
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
merge bot Applied
#3 lore raw
In reply to Marc Zyngier
The following commit has been merged into the irq/core branch of tip:

The following commit has been merged into the irq/core branch of tip:

Commit-ID:     64b6d1d7a84538de34c22a6fc92a7dcc2b196b64
Gitweb:        https://git.kernel.org/tip/64b6d1d7a84538de34c22a6fc92a7dcc2b196b64
Author:        Marc Zyngier <[email protected]>
AuthorDate:    Mon, 26 Aug 2024 09:06:18 +01:00
Committer:     Thomas Gleixner <[email protected]>
CommitterDate: Tue, 27 Aug 2024 13:54:15 +02:00

genirq: Get rid of global lock in irq_do_set_affinity()

Kunkun Jiang reports that for a workload involving the simultaneous startup of a large number of VMs (for a total of about 200 vcpus), a lot of CPU time gets spent on spinning on the tmp_mask_lock that exists as a static raw spinlock in irq_do_set_affinity(). This lock protects a global cpumask (tmp_mask) that is used as a temporary variable to compute the resulting affinity.

While this is triggered by KVM issuing a irq_set_affinity() call each time a vcpu is about to execute, it is obvious that having a single global resource is not very scalable.

Since a cpumask can be a fairly large structure on systems with a high core count, a stack allocation is not really appropriate. Instead, turn the global cpumask into a per-CPU variable, removing the need for locking altogether as the code is executed with preemption and interrupts disabled.

[ tglx: Moved the per CPU variable declaration outside of the function ]

---
 kernel/irq/manage.c | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)
kernel/irq/manage.c +9 -12
index dd53298..f0803d6 100644
@@ -218,21 +218,20 @@ static void irq_validate_effective_affinity(struct irq_data *data)
 static inline void irq_validate_effective_affinity(struct irq_data *data) { }
 #endif
 
+static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
+
 int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,
 			bool force)
 {
+	struct cpumask *tmp_mask = this_cpu_ptr(&__tmp_mask);
 	struct irq_desc *desc = irq_data_to_desc(data);
 	struct irq_chip *chip = irq_data_get_irq_chip(data);
 	const struct cpumask  *prog_mask;
 	int ret;
 
-	static DEFINE_RAW_SPINLOCK(tmp_mask_lock);
-	static struct cpumask tmp_mask;
-
 	if (!chip || !chip->irq_set_affinity)
 		return -EINVAL;
 
-	raw_spin_lock(&tmp_mask_lock);
 	/*
 	 * If this is a managed interrupt and housekeeping is enabled on
 	 * it check whether the requested affinity mask intersects with
@@ -258,11 +257,11 @@ int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,
 
 		hk_mask = housekeeping_cpumask(HK_TYPE_MANAGED_IRQ);
 
-		cpumask_and(&tmp_mask, mask, hk_mask);
-		if (!cpumask_intersects(&tmp_mask, cpu_online_mask))
+		cpumask_and(tmp_mask, mask, hk_mask);
+		if (!cpumask_intersects(tmp_mask, cpu_online_mask))
 			prog_mask = mask;
 		else
-			prog_mask = &tmp_mask;
+			prog_mask = tmp_mask;
 	} else {
 		prog_mask = mask;
 	}
@@ -272,16 +271,14 @@ int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,
 	 * unless we are being asked to force the affinity (in which
 	 * case we do as we are told).
 	 */
-	cpumask_and(&tmp_mask, prog_mask, cpu_online_mask);
-	if (!force && !cpumask_empty(&tmp_mask))
-		ret = chip->irq_set_affinity(data, &tmp_mask, force);
+	cpumask_and(tmp_mask, prog_mask, cpu_online_mask);
+	if (!force && !cpumask_empty(tmp_mask))
+		ret = chip->irq_set_affinity(data, tmp_mask, force);
 	else if (force)
 		ret = chip->irq_set_affinity(data, mask, force);
 	else
 		ret = -EINVAL;
 
-	raw_spin_unlock(&tmp_mask_lock);
-
 	switch (ret) {
 	case IRQ_SET_MASK_OK:
 	case IRQ_SET_MASK_OK_DONE: