genirq: Get rid of global lock in irq_do_set_affinity()
PATCH 1/1 genirq: Get rid of global lock in irq_do_set_affinity()
Kunkun Jiang reports that for a workload involving the simultaneous startup of a large number of VMs (for a total of about 200 vcpus), a lot of CPU time gets spent on spinning on the tmp_mask_lock that exists as a static raw spinlock in irq_do_set_affinity(). This lock protects a global cpumask (tmp_mask) that is used as a temporary variable to compute the resulting affinity.
While this is triggered by KVM issuing a irq_set_affinity() call each time a vcpu is about to execute, it is obvious that having a single global resource is not very scalable, and that we could do something better.
Since a cpumask can be a fairly large structure on systems with a high core count, a stack allocation is not really appropriate. Instead, turn the global cpumask into a per-CPU variable, removing the need for locking altogether as we are not preemptible at this point.
--- kernel/irq/manage.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-)
kernel/irq/manage.c +10 -11
@@ -224,15 +224,16 @@ int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,struct irq_desc *desc =irq_data_to_desc (data);struct irq_chip *chip =irq_data_get_irq_chip (data);const struct cpumask *prog_mask; +struct cpumask *tmp_mask;int ret; -static DEFINE_RAW_SPINLOCK (tmp_mask_lock); -static struct cpumask tmp_mask; + static DEFINE_PER_CPU(struct cpumask, __tmp_mask);if (!chip || !chip->irq_set_affinity)return -EINVAL ; -raw_spin_lock (&tmp_mask_lock); + tmp_mask =this_cpu_ptr (&__tmp_mask); +/* * If this is a managed interrupt and housekeeping is enabled on * it check whether the requested affinity mask intersects with
@@ -258,11 +259,11 @@ int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask, hk_mask =housekeeping_cpumask (HK_TYPE_MANAGED_IRQ ); -cpumask_and (&tmp_mask, mask, hk_mask); -if (!cpumask_intersects (&tmp_mask, cpu_online_mask)) +cpumask_and (tmp_mask, mask, hk_mask); +if (!cpumask_intersects (tmp_mask, cpu_online_mask)) prog_mask = mask;else - prog_mask = &tmp_mask; + prog_mask = tmp_mask; }else { prog_mask = mask; }
@@ -272,16 +273,14 @@ int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,* unless we are being asked to force the affinity (in which * case we do as we are told). */ -cpumask_and (&tmp_mask, prog_mask, cpu_online_mask); -if (!force && !cpumask_empty (&tmp_mask)) - ret = chip->irq_set_affinity (data, &tmp_mask, force); +cpumask_and (tmp_mask, prog_mask, cpu_online_mask); +if (!force && !cpumask_empty (tmp_mask)) + ret = chip->irq_set_affinity (data, tmp_mask, force);else if (force) ret = chip->irq_set_affinity (data, mask, force);else ret = -EINVAL ; -raw_spin_unlock (&tmp_mask_lock); -switch (ret) {case IRQ_SET_MASK_OK :case IRQ_SET_MASK_OK_DONE :
Signature · 1 line
2.39.2
The following commit has been merged into the irq/core branch of tip:
The following commit has been merged into the irq/core branch of tip:
Commit-ID: 6c70d79f363c0c9a712e107b9c4d3644ca8c7490 Gitweb: https://git.kernel.org/tip/6c70d79f363c0c9a712e107b9c4d3644ca8c7490 Author: Marc Zyngier <[email protected]> AuthorDate: Mon, 26 Aug 2024 09:06:18 +01:00 Committer: Thomas Gleixner <[email protected]> CommitterDate: Mon, 26 Aug 2024 12:52:35 +02:00
genirq: Get rid of global lock in irq_do_set_affinity()
Kunkun Jiang reports that for a workload involving the simultaneous startup of a large number of VMs (for a total of about 200 vcpus), a lot of CPU time gets spent on spinning on the tmp_mask_lock that exists as a static raw spinlock in irq_do_set_affinity(). This lock protects a global cpumask (tmp_mask) that is used as a temporary variable to compute the resulting affinity.
While this is triggered by KVM issuing a irq_set_affinity() call each time a vcpu is about to execute, it is obvious that having a single global resource is not very scalable.
Since a cpumask can be a fairly large structure on systems with a high core count, a stack allocation is not really appropriate. Instead, turn the global cpumask into a per-CPU variable, removing the need for locking altogether as the code is executed with preemption and interrupts disabled.
--- kernel/irq/manage.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-)
kernel/irq/manage.c +10 -11
@@ -224,15 +224,16 @@ int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,struct irq_desc *desc =irq_data_to_desc (data);struct irq_chip *chip =irq_data_get_irq_chip (data);const struct cpumask *prog_mask; +struct cpumask *tmp_mask;int ret; -static DEFINE_RAW_SPINLOCK (tmp_mask_lock); -static struct cpumask tmp_mask; + static DEFINE_PER_CPU(struct cpumask, __tmp_mask);if (!chip || !chip->irq_set_affinity)return -EINVAL ; -raw_spin_lock (&tmp_mask_lock); + tmp_mask =this_cpu_ptr (&__tmp_mask); +/* * If this is a managed interrupt and housekeeping is enabled on * it check whether the requested affinity mask intersects with
@@ -258,11 +259,11 @@ int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask, hk_mask =housekeeping_cpumask (HK_TYPE_MANAGED_IRQ ); -cpumask_and (&tmp_mask, mask, hk_mask); -if (!cpumask_intersects (&tmp_mask, cpu_online_mask)) +cpumask_and (tmp_mask, mask, hk_mask); +if (!cpumask_intersects (tmp_mask, cpu_online_mask)) prog_mask = mask;else - prog_mask = &tmp_mask; + prog_mask = tmp_mask; }else { prog_mask = mask; }
@@ -272,16 +273,14 @@ int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,* unless we are being asked to force the affinity (in which * case we do as we are told). */ -cpumask_and (&tmp_mask, prog_mask, cpu_online_mask); -if (!force && !cpumask_empty (&tmp_mask)) - ret = chip->irq_set_affinity (data, &tmp_mask, force); +cpumask_and (tmp_mask, prog_mask, cpu_online_mask); +if (!force && !cpumask_empty (tmp_mask)) + ret = chip->irq_set_affinity (data, tmp_mask, force);else if (force) ret = chip->irq_set_affinity (data, mask, force);else ret = -EINVAL ; -raw_spin_unlock (&tmp_mask_lock); -switch (ret) {case IRQ_SET_MASK_OK :case IRQ_SET_MASK_OK_DONE :
kernel test robot
build botkernel test robot noticed the following build errors:
Hi Marc,
kernel test robot noticed the following build errors:
[auto build test ERROR on tip/irq/core]
[also build test ERROR on linus/master v6.11-rc5 next-20240826] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Marc-Zyngier/genirq-Get-rid-of-global-lock-in-irq_do_set_affinity/20240826-170538 base: tip/irq/core patch link: https://lore.kernel.org/r/20240826080618.3886694-1-maz%40kernel.org patch subject: [PATCH] genirq: Get rid of global lock in irq_do_set_affinity() config: arc-randconfig-002-20240827 (https://download.01.org/0day-ci/archive/20240827/[email protected]/config) compiler: arceb-elf-gcc (GCC) 13.2.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20240827/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All errors (new ones prefixed by >>):
In file included from include/asm-generic/percpu.h:7,
from ./arch/arc/include/generated/asm/percpu.h:1,
from include/linux/irqflags.h:19,
from arch/arc/include/asm/smp.h:104,
from arch/arc/include/asm/cmpxchg.h:13,
from arch/arc/include/asm/atomic.h:13,
from include/linux/atomic.h:7,
from include/asm-generic/bitops/lock.h:5,
from arch/arc/include/asm/bitops.h:188,
from include/linux/bitops.h:68,
from include/linux/thread_info.h:27,
from include/asm-generic/preempt.h:5,
from ./arch/arc/include/generated/asm/preempt.h:1,
from include/linux/preempt.h:79,
from include/linux/spinlock.h:56,
from include/linux/irq.h:14,
from kernel/irq/manage.c:11:
kernel/irq/manage.c: In function 'irq_do_set_affinity':
include/linux/percpu-defs.h:92:40: error: section attribute cannot be specified for local variables
92 | extern __PCPU_DUMMY_ATTRS char __pcpu_unique_##name; \
| ^~~~~~~~~~~~~~
include/linux/percpu-defs.h:115:9: note: in expansion of macro 'DEFINE_PER_CPU_SECTION'
115 | DEFINE_PER_CPU_SECTION(type, name, "")
| ^~~~~~~~~~~~~~~~~~~~~~
kernel/irq/manage.c:230:16: note: in expansion of macro 'DEFINE_PER_CPU'
230 | static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
| ^~~~~~~~~~~~~~
include/linux/percpu-defs.h:93:33: error: section attribute cannot be specified for local variables
93 | __PCPU_DUMMY_ATTRS char __pcpu_unique_##name; \
| ^~~~~~~~~~~~~~
include/linux/percpu-defs.h:115:9: note: in expansion of macro 'DEFINE_PER_CPU_SECTION'
115 | DEFINE_PER_CPU_SECTION(type, name, "")
| ^~~~~~~~~~~~~~~~~~~~~~
kernel/irq/manage.c:230:16: note: in expansion of macro 'DEFINE_PER_CPU'
230 | static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
| ^~~~~~~~~~~~~~
>> include/linux/percpu-defs.h:93:33: error: declaration of '__pcpu_unique___tmp_mask' with no linkage follows extern declaration
93 | __PCPU_DUMMY_ATTRS char __pcpu_unique_##name; \
| ^~~~~~~~~~~~~~
include/linux/percpu-defs.h:115:9: note: in expansion of macro 'DEFINE_PER_CPU_SECTION'
115 | DEFINE_PER_CPU_SECTION(type, name, "")
| ^~~~~~~~~~~~~~~~~~~~~~
kernel/irq/manage.c:230:16: note: in expansion of macro 'DEFINE_PER_CPU'
230 | static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
| ^~~~~~~~~~~~~~
include/linux/percpu-defs.h:92:40: note: previous declaration of '__pcpu_unique___tmp_mask' with type 'char'
92 | extern __PCPU_DUMMY_ATTRS char __pcpu_unique_##name; \
| ^~~~~~~~~~~~~~
include/linux/percpu-defs.h:115:9: note: in expansion of macro 'DEFINE_PER_CPU_SECTION'
115 | DEFINE_PER_CPU_SECTION(type, name, "")
| ^~~~~~~~~~~~~~~~~~~~~~
kernel/irq/manage.c:230:16: note: in expansion of macro 'DEFINE_PER_CPU'
230 | static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
| ^~~~~~~~~~~~~~
>> kernel/irq/manage.c:230:47: error: section attribute cannot be specified for local variables
230 | static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
| ^~~~~~~~~~
include/linux/percpu-defs.h:94:51: note: in definition of macro 'DEFINE_PER_CPU_SECTION'
94 | extern __PCPU_ATTRS(sec) __typeof__(type) name; \
| ^~~~
kernel/irq/manage.c:230:16: note: in expansion of macro 'DEFINE_PER_CPU'
230 | static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
| ^~~~~~~~~~~~~~
>> kernel/irq/manage.c:230:47: error: section attribute cannot be specified for local variables
230 | static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
| ^~~~~~~~~~
include/linux/percpu-defs.h:95:51: note: in definition of macro 'DEFINE_PER_CPU_SECTION'
95 | __PCPU_ATTRS(sec) __weak __typeof__(type) name
| ^~~~
kernel/irq/manage.c:230:16: note: in expansion of macro 'DEFINE_PER_CPU'
230 | static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
| ^~~~~~~~~~~~~~
>> kernel/irq/manage.c:230:47: error: weak declaration of '__tmp_mask' must be public
230 | static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
| ^~~~~~~~~~
include/linux/percpu-defs.h:95:51: note: in definition of macro 'DEFINE_PER_CPU_SECTION'
95 | __PCPU_ATTRS(sec) __weak __typeof__(type) name
| ^~~~
kernel/irq/manage.c:230:16: note: in expansion of macro 'DEFINE_PER_CPU'
230 | static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
| ^~~~~~~~~~~~~~
>> kernel/irq/manage.c:230:47: error: declaration of '__tmp_mask' with no linkage follows extern declaration
230 | static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
| ^~~~~~~~~~
include/linux/percpu-defs.h:95:51: note: in definition of macro 'DEFINE_PER_CPU_SECTION'
95 | __PCPU_ATTRS(sec) __weak __typeof__(type) name
| ^~~~
kernel/irq/manage.c:230:16: note: in expansion of macro 'DEFINE_PER_CPU'
230 | static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
| ^~~~~~~~~~~~~~
kernel/irq/manage.c:230:47: note: previous declaration of '__tmp_mask' with type 'struct cpumask'
230 | static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
| ^~~~~~~~~~
include/linux/percpu-defs.h:94:51: note: in definition of macro 'DEFINE_PER_CPU_SECTION'
94 | extern __PCPU_ATTRS(sec) __typeof__(type) name; \
| ^~~~
kernel/irq/manage.c:230:16: note: in expansion of macro 'DEFINE_PER_CPU'
230 | static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
| ^~~~~~~~~~~~~~vim +230 kernel/irq/manage.c
220
221 int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,
222 bool force)
223 {
224 struct irq_desc *desc = irq_data_to_desc(data);
225 struct irq_chip *chip = irq_data_get_irq_chip(data);
226 const struct cpumask *prog_mask;
227 struct cpumask *tmp_mask;
228 int ret;
229
> 230 static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
231
232 if (!chip || !chip->irq_set_affinity)
233 return -EINVAL;
234
235 tmp_mask = this_cpu_ptr(&__tmp_mask);
236
237 /*
238 * If this is a managed interrupt and housekeeping is enabled on
239 * it check whether the requested affinity mask intersects with
240 * a housekeeping CPU. If so, then remove the isolated CPUs from
241 * the mask and just keep the housekeeping CPU(s). This prevents
242 * the affinity setter from routing the interrupt to an isolated
243 * CPU to avoid that I/O submitted from a housekeeping CPU causes
244 * interrupts on an isolated one.
245 *
246 * If the masks do not intersect or include online CPU(s) then
247 * keep the requested mask. The isolated target CPUs are only
248 * receiving interrupts when the I/O operation was submitted
249 * directly from them.
250 *
251 * If all housekeeping CPUs in the affinity mask are offline, the
252 * interrupt will be migrated by the CPU hotplug code once a
253 * housekeeping CPU which belongs to the affinity mask comes
254 * online.
255 */
256 if (irqd_affinity_is_managed(data) &&
257 housekeeping_enabled(HK_TYPE_MANAGED_IRQ)) {
258 const struct cpumask *hk_mask;
259
260 hk_mask = housekeeping_cpumask(HK_TYPE_MANAGED_IRQ);
261
262 cpumask_and(tmp_mask, mask, hk_mask);
263 if (!cpumask_intersects(tmp_mask, cpu_online_mask))
264 prog_mask = mask;
265 else
266 prog_mask = tmp_mask;
267 } else {
268 prog_mask = mask;
269 }
270
271 /*
272 * Make sure we only provide online CPUs to the irqchip,
273 * unless we are being asked to force the affinity (in which
274 * case we do as we are told).
275 */
276 cpumask_and(tmp_mask, prog_mask, cpu_online_mask);
277 if (!force && !cpumask_empty(tmp_mask))
278 ret = chip->irq_set_affinity(data, tmp_mask, force);
279 else if (force)
280 ret = chip->irq_set_affinity(data, mask, force);
281 else
282 ret = -EINVAL;
283
284 switch (ret) {
285 case IRQ_SET_MASK_OK:
286 case IRQ_SET_MASK_OK_DONE:
287 cpumask_copy(desc->irq_common_data.affinity, mask);
288 fallthrough;
289 case IRQ_SET_MASK_OK_NOCOPY:
290 irq_validate_effective_affinity(data);
291 irq_set_thread_affinity(desc);
292 ret = 0;
293 }
294
295 return ret;
296 }
297
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
The following commit has been merged into the irq/core branch of tip:
The following commit has been merged into the irq/core branch of tip:
Commit-ID: 64b6d1d7a84538de34c22a6fc92a7dcc2b196b64 Gitweb: https://git.kernel.org/tip/64b6d1d7a84538de34c22a6fc92a7dcc2b196b64 Author: Marc Zyngier <[email protected]> AuthorDate: Mon, 26 Aug 2024 09:06:18 +01:00 Committer: Thomas Gleixner <[email protected]> CommitterDate: Tue, 27 Aug 2024 13:54:15 +02:00
genirq: Get rid of global lock in irq_do_set_affinity()
Kunkun Jiang reports that for a workload involving the simultaneous startup of a large number of VMs (for a total of about 200 vcpus), a lot of CPU time gets spent on spinning on the tmp_mask_lock that exists as a static raw spinlock in irq_do_set_affinity(). This lock protects a global cpumask (tmp_mask) that is used as a temporary variable to compute the resulting affinity.
While this is triggered by KVM issuing a irq_set_affinity() call each time a vcpu is about to execute, it is obvious that having a single global resource is not very scalable.
Since a cpumask can be a fairly large structure on systems with a high core count, a stack allocation is not really appropriate. Instead, turn the global cpumask into a per-CPU variable, removing the need for locking altogether as the code is executed with preemption and interrupts disabled.
[ tglx: Moved the per CPU variable declaration outside of the function ]
--- kernel/irq/manage.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-)
kernel/irq/manage.c +9 -12
@@ -218,21 +218,20 @@ static void irq_validate_effective_affinity(struct irq_data *data)static inline void irq_validate_effective_affinity (struct irq_data *data) { }#endif +static DEFINE_PER_CPU(struct cpumask, __tmp_mask); +int irq_do_set_affinity (struct irq_data *data,const struct cpumask *mask,bool force) { +struct cpumask *tmp_mask =this_cpu_ptr (&__tmp_mask);struct irq_desc *desc =irq_data_to_desc (data);struct irq_chip *chip =irq_data_get_irq_chip (data);const struct cpumask *prog_mask;int ret; -static DEFINE_RAW_SPINLOCK (tmp_mask_lock); -static struct cpumask tmp_mask; -if (!chip || !chip->irq_set_affinity)return -EINVAL ; -raw_spin_lock (&tmp_mask_lock);/* * If this is a managed interrupt and housekeeping is enabled on * it check whether the requested affinity mask intersects with
@@ -258,11 +257,11 @@ int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask, hk_mask =housekeeping_cpumask (HK_TYPE_MANAGED_IRQ ); -cpumask_and (&tmp_mask, mask, hk_mask); -if (!cpumask_intersects (&tmp_mask, cpu_online_mask)) +cpumask_and (tmp_mask, mask, hk_mask); +if (!cpumask_intersects (tmp_mask, cpu_online_mask)) prog_mask = mask;else - prog_mask = &tmp_mask; + prog_mask = tmp_mask; }else { prog_mask = mask; }
@@ -272,16 +271,14 @@ int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,* unless we are being asked to force the affinity (in which * case we do as we are told). */ -cpumask_and (&tmp_mask, prog_mask, cpu_online_mask); -if (!force && !cpumask_empty (&tmp_mask)) - ret = chip->irq_set_affinity (data, &tmp_mask, force); +cpumask_and (tmp_mask, prog_mask, cpu_online_mask); +if (!force && !cpumask_empty (tmp_mask)) + ret = chip->irq_set_affinity (data, tmp_mask, force);else if (force) ret = chip->irq_set_affinity (data, mask, force);else ret = -EINVAL ; -raw_spin_unlock (&tmp_mask_lock); -switch (ret) {case IRQ_SET_MASK_OK :case IRQ_SET_MASK_OK_DONE :