nohz: Nohz full kick fixes
Subsystems (20)
cover nohz: Nohz full kick fixes
Hi,
There have been several reports of warnings and lockup lately due to recent or longer term issues.
This patchset is a proposed solution to fix them.
git://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git
nohz/fixes
Thanks,
Frederic
---
Frederic Weisbecker (9):
nohz: Restore NMI safe local irq work for local nohz kick
nohz: Move nohz full init call to tick init
irq_work: Force raised irq work to run on irq work interrupt
x86: Build irq work only if local apic support
x86: Tell irq work about self IPI support
arm: Tell irq work about self IPI support
arm64: Tell irq work about self IPI support
nohz: Consolidate nohz full init code
nohz: nohz full depends on irq work self IPI support
arch/arm/kernel/smp.c | 5 +++ arch/arm64/kernel/smp.c | 5 +++ arch/x86/Kconfig | 4 +++ arch/x86/include/asm/entry_arch.h | 2 +- arch/x86/kernel/Makefile | 2 +- arch/x86/kernel/entry_64.S | 2 +- arch/x86/kernel/irq_work.c | 7 +++-- arch/x86/kernel/irqinit.c | 2 +- include/linux/irq_work.h | 4 +++ include/linux/tick.h | 9 +----- init/main.c | 2 +- kernel/irq_work.c | 21 ++++++++++++- kernel/time/tick-common.c | 1 + kernel/time/tick-internal.h | 7 +++++ kernel/time/tick-sched.c | 66 ++++++++++++++++++++++++++++----------- kernel/time/timer.c | 2 +- 16 files changed, 105 insertions(+), 36 deletions(-)
RFC PATCH 1/9 nohz: Restore NMI safe local irq work for local nohz kick
The local nohz kick is currently used by perf which needs it to be NMI-safe. Recent commit though (7d1311b93e58ed55f3a31cc8f94c4b8fe988a2b9) changed its implementation to fire the local kick using the remote kick API. It was convenient to make the code more generic but the remote kick isn't NMI-safe.
As a result:
WARNING: CPU: 3 PID: 18062 at kernel/irq_work.c:72 irq_work_queue_on+0x11e/0x140() CPU: 3 PID: 18062 Comm: trinity-subchil Not tainted 3.16.0+ #34 0000000000000009 00000000903774d1 ffff880244e06c00 ffffffff9a7f1e37 0000000000000000 ffff880244e06c38 ffffffff9a0791dd ffff880244fce180 0000000000000003 ffff880244e06d58 ffff880244e06ef8 0000000000000000 Call Trace: <NMI> [<ffffffff9a7f1e37>] dump_stack+0x4e/0x7a [<ffffffff9a0791dd>] warn_slowpath_common+0x7d/0xa0 [<ffffffff9a07930a>] warn_slowpath_null+0x1a/0x20 [<ffffffff9a17ca1e>] irq_work_queue_on+0x11e/0x140 [<ffffffff9a10a2c7>] tick_nohz_full_kick_cpu+0x57/0x90 [<ffffffff9a186cd5>] __perf_event_overflow+0x275/0x350 [<ffffffff9a184f80>] ? perf_event_task_disable+0xa0/0xa0 [<ffffffff9a01a4cf>] ? x86_perf_event_set_period+0xbf/0x150 [<ffffffff9a187934>] perf_event_overflow+0x14/0x20 [<ffffffff9a020386>] intel_pmu_handle_irq+0x206/0x410 [<ffffffff9a0b54d3>] ? arch_vtime_task_switch+0x63/0x130 [<ffffffff9a01937b>] perf_event_nmi_handler+0x2b/0x50 [<ffffffff9a007b72>] nmi_handle+0xd2/0x390 [<ffffffff9a007aa5>] ? nmi_handle+0x5/0x390 [<ffffffff9a0d131b>] ? lock_release+0xab/0x330 [<ffffffff9a008062>] default_do_nmi+0x72/0x1c0 [<ffffffff9a0c925f>] ? cpuacct_account_field+0xcf/0x200 [<ffffffff9a008268>] do_nmi+0xb8/0x100
Lets fix this by restoring the use of local irq work for the nohz local kick.
--- include/linux/tick.h | 7 +------ kernel/time/tick-sched.c | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 6 deletions(-)
include/linux/tick.h +1 -6
@@ -183,13 +183,8 @@ static inline bool tick_nohz_full_cpu(int cpu)extern void tick_nohz_init (void );extern void __tick_nohz_full_check (void ); +extern void tick_nohz_full_kick (void );extern void tick_nohz_full_kick_cpu (int cpu); - -static inline void tick_nohz_full_kick (void ) -{ -tick_nohz_full_kick_cpu (smp_processor_id ()); -} -extern void tick_nohz_full_kick_all (void );extern void __tick_nohz_task_switch (struct task_struct *tsk);#else
kernel/time/tick-sched.c +14 -0
@@ -225,6 +225,20 @@ static DEFINE_PER_CPU(struct irq_work, nohz_full_kick_work) = { };/* +* Kick this CPU if it's full dynticks in order to force it to +* re-evaluate its dependency on the tick and restart it if necessary. +* This kick, unlike tick_nohz_full_kick_cpu() and tick_nohz_full_kick_all(), +* is NMI safe. +*/ +void tick_nohz_full_kick (void ) +{ +if (!tick_nohz_full_cpu (smp_processor_id ())) +return ; + +irq_work_queue (&__get_cpu_var (nohz_full_kick_work)); +} + +/* * Kick the CPU if it's full dynticks in order to force it to * re-evaluate its dependency on the tick and restart it if necessary. */
Signature · 1 line
1.8.3.1
RFC PATCH 2/9 nohz: Move nohz full init call to tick init
In reply to Frederic WeisbeckerThis way we unbloat a bit main.c and more importantly we initialize nohz full after init_IRQ(). This dependency will be needed in further patches because nohz full depends on irq work to raise its own IRQ. This ability, on ARM64, is obtained only after init_IRQ() which initialize the pointer to __smp_call_function.
--- include/linux/tick.h | 2 -- init/main.c | 1 - kernel/time/tick-common.c | 1 + kernel/time/tick-internal.h | 7 +++++++ 4 files changed, 8 insertions(+), 3 deletions(-)
include/linux/tick.h +0 -2
@@ -181,14 +181,12 @@ static inline bool tick_nohz_full_cpu(int cpu) return cpumask_test_cpu(cpu, tick_nohz_full_mask); } -extern void tick_nohz_init (void );extern void __tick_nohz_full_check (void );extern void tick_nohz_full_kick (void );extern void tick_nohz_full_kick_cpu (int cpu);extern void tick_nohz_full_kick_all (void );extern void __tick_nohz_task_switch (struct task_struct *tsk);#else -static inline void tick_nohz_init (void ) { }static inline bool tick_nohz_full_enabled (void ) {return false; }static inline bool tick_nohz_full_cpu (int cpu) {return false; }static inline void __tick_nohz_full_check (void ) { }
init/main.c +0 -1
@@ -577,7 +577,6 @@ asmlinkage __visible void __init start_kernel(void)local_irq_disable ();idr_init_cache ();rcu_init (); -tick_nohz_init ();context_tracking_init ();radix_tree_init ();/* init some links before init_ISA_irqs() */
kernel/time/tick-common.c +1 -0
@@ -400,4 +400,5 @@ void tick_resume(void)void __init tick_init (void ) {tick_broadcast_init (); +tick_nohz_init (); }
kernel/time/tick-internal.h +7 -0
@@ -99,6 +99,13 @@ static inline int tick_broadcast_oneshot_active(void) { return 0; }static inline bool tick_broadcast_oneshot_available (void ) {return false; }#endif /* !TICK_ONESHOT */ +/* NO_HZ_FULL internal */ +#ifdef CONFIG_NO_HZ_FULL +extern void tick_nohz_init (void ); +# else +static inline void tick_nohz_init (void ) { } +#endif +/* * Broadcasting support */
Signature · 1 line
1.8.3.1
RFC PATCH 3/9 irq_work: Force raised irq work to run on irq work interrupt
In reply to Frederic WeisbeckerThe nohz full kick, which restarts the tick after some new state depend on it, can't be called from any place given the operation it does on timers. If it is called from the scheduler or timers code, chances are that we run into a deadlock.
This is why we run the nohz full kick from an irq work. That way we make sure that the kick runs on a virgin context.
However if that's the case when irq work runs in its own dedicated self-ipi, things are different for the big bunch of archs that don't support the self triggered way. In order to handle them, irq works are also handled by the timer interrupt as fallback.
Now when irq works run on the timer interrupt, the context isn't blank. More precisely, they can run in the context of the hrtimer that runs the tick. But the nohz kick cancels and restarts this hrtimer. If it does the cancellation from the hrtimer itself, we run in an endless loop:
Kernel panic - not syncing: Watchdog detected hard LOCKUP on cpu 2 CPU: 2 PID: 7538 Comm: kworker/u8:8 Not tainted 3.16.0+ #34 Workqueue: btrfs-endio-write normal_work_helper [btrfs] ffff880244c06c88 000000001b486fe1 ffff880244c06bf0 ffffffff8a7f1e37 ffffffff8ac52a18 ffff880244c06c78 ffffffff8a7ef928 0000000000000010 ffff880244c06c88 ffff880244c06c20 000000001b486fe1 0000000000000000 Call Trace: <NMI[<ffffffff8a7f1e37>] dump_stack+0x4e/0x7a [<ffffffff8a7ef928>] panic+0xd4/0x207 [<ffffffff8a1450e8>] watchdog_overflow_callback+0x118/0x120 [<ffffffff8a186b0e>] __perf_event_overflow+0xae/0x350 [<ffffffff8a184f80>] ? perf_event_task_disable+0xa0/0xa0 [<ffffffff8a01a4cf>] ? x86_perf_event_set_period+0xbf/0x150 [<ffffffff8a187934>] perf_event_overflow+0x14/0x20 [<ffffffff8a020386>] intel_pmu_handle_irq+0x206/0x410 [<ffffffff8a01937b>] perf_event_nmi_handler+0x2b/0x50 [<ffffffff8a007b72>] nmi_handle+0xd2/0x390 [<ffffffff8a007aa5>] ? nmi_handle+0x5/0x390 [<ffffffff8a0cb7f8>] ? match_held_lock+0x8/0x1b0 [<ffffffff8a008062>] default_do_nmi+0x72/0x1c0 [<ffffffff8a008268>] do_nmi+0xb8/0x100 [<ffffffff8a7ff66a>] end_repeat_nmi+0x1e/0x2e [<ffffffff8a0cb7f8>] ? match_held_lock+0x8/0x1b0 [<ffffffff8a0cb7f8>] ? match_held_lock+0x8/0x1b0 [<ffffffff8a0cb7f8>] ? match_held_lock+0x8/0x1b0 <<EOE><IRQ[<ffffffff8a0ccd2f>] lock_acquired+0xaf/0x450 [<ffffffff8a0f74c5>] ? lock_hrtimer_base.isra.20+0x25/0x50 [<ffffffff8a7fc678>] _raw_spin_lock_irqsave+0x78/0x90 [<ffffffff8a0f74c5>] ? lock_hrtimer_base.isra.20+0x25/0x50 [<ffffffff8a0f74c5>] lock_hrtimer_base.isra.20+0x25/0x50 [<ffffffff8a0f7723>] hrtimer_try_to_cancel+0x33/0x1e0 [<ffffffff8a0f78ea>] hrtimer_cancel+0x1a/0x30 [<ffffffff8a109237>] tick_nohz_restart+0x17/0x90 [<ffffffff8a10a213>] __tick_nohz_full_check+0xc3/0x100 [<ffffffff8a10a25e>] nohz_full_kick_work_func+0xe/0x10 [<ffffffff8a17c884>] irq_work_run_list+0x44/0x70 [<ffffffff8a17c8da>] irq_work_run+0x2a/0x50 [<ffffffff8a0f700b>] update_process_times+0x5b/0x70 [<ffffffff8a109005>] tick_sched_handle.isra.21+0x25/0x60 [<ffffffff8a109b81>] tick_sched_timer+0x41/0x60 [<ffffffff8a0f7aa2>] __run_hrtimer+0x72/0x470 [<ffffffff8a109b40>] ? tick_sched_do_timer+0xb0/0xb0 [<ffffffff8a0f8707>] hrtimer_interrupt+0x117/0x270 [<ffffffff8a034357>] local_apic_timer_interrupt+0x37/0x60 [<ffffffff8a80010f>] smp_apic_timer_interrupt+0x3f/0x50 [<ffffffff8a7fe52f>] apic_timer_interrupt+0x6f/0x80
Two possible solutions to fix this: either we make the kick self-aware of the context it runs on (tick hrtimer or irq work IPI) and adapt the action on top of it or we force non-lazy irq works to run on irq work IPIs when available.
The second solution needs the irq work subsystem to know if the arch supports irq work IPIs. This information will have to be available anyway because nohz full can't work without that arch support and we must initialize irq work properly with that dependency in mind.
So lets provide a way for archs to tell about that irq work IPI support. On top of that we can force irq works on that IPI to avoid lockups like above.
--- include/linux/irq_work.h | 3 +++ init/main.c | 1 + kernel/irq_work.c | 21 ++++++++++++++++++++- kernel/time/timer.c | 2 +- 4 files changed, 25 insertions(+), 2 deletions(-)
include/linux/irq_work.h +3 -0
@@ -39,11 +39,14 @@ bool irq_work_queue_on(struct irq_work *work, int cpu);#endif void irq_work_run (void ); +void irq_work_tick (void );void irq_work_sync (struct irq_work *work);#ifdef CONFIG_IRQ_WORK +void irq_work_init (void );bool irq_work_needs_cpu (void );#else +static inline void irq_work_init (void ) { }static inline bool irq_work_needs_cpu (void ) {return false; }#endif
init/main.c +1 -0
@@ -582,6 +582,7 @@ asmlinkage __visible void __init start_kernel(void)/* init some links before init_ISA_irqs() */ early_irq_init ();init_IRQ (); +irq_work_init ();tick_init ();init_timers ();hrtimers_init ();
kernel/irq_work.c +20 -1
@@ -22,6 +22,7 @@ static DEFINE_PER_CPU(struct llist_head, raised_list); static DEFINE_PER_CPU(struct llist_head, lazy_list); +static bool has_own_interrupt;/* * Claim the entry so that no one else will poke at it.
@@ -56,6 +57,11 @@ void __weak arch_irq_work_raise(void)*/ } +int __init __weak arch_irq_work_has_own_interrupt (void ) +{ +return 0 ; +} +#ifdef CONFIG_SMP /* * Enqueue the irq_work @work on @cpu unless it's already pending
@@ -115,7 +121,8 @@ bool irq_work_needs_cpu(void) raised = &__get_cpu_var (raised_list); lazy = &__get_cpu_var (lazy_list); -if (llist_empty (raised) &&llist_empty (lazy)) + +if ((llist_empty (raised) || has_own_interrupt) &&llist_empty (lazy))return false;/* All work should have been flushed before going offline */
@@ -171,6 +178,13 @@ void irq_work_run(void) }EXPORT_SYMBOL_GPL (irq_work_run); +void irq_work_tick (void ) +{ +if (!has_own_interrupt) +irq_work_run_list (&__get_cpu_var (raised_list)); +irq_work_run_list (&__get_cpu_var (lazy_list)); +} +/* * Synchronize against the irq_work @entry, ensures the entry is not * currently in use.
@@ -183,3 +197,8 @@ void irq_work_sync(struct irq_work *work)cpu_relax (); }EXPORT_SYMBOL_GPL (irq_work_sync); + +void __init irq_work_init (void ) +{ + has_own_interrupt =arch_irq_work_has_own_interrupt (); +}
kernel/time/timer.c +1 -1
@@ -1385,7 +1385,7 @@ void update_process_times(int user_tick)rcu_check_callbacks (cpu, user_tick);#ifdef CONFIG_IRQ_WORK if (in_irq ()) -irq_work_run (); +irq_work_tick ();#endif scheduler_tick ();run_posix_cpu_timers (p);
Signature · 1 line
1.8.3.1
Re: [RFC PATCH 3/9] irq_work: Force raised irq work to run on irq work interrupt
Quoted 13 lines · Frederic Weisbecker
kernel/irq_work.c +1
-0@@ -22,6 +22,7 @@ static DEFINE_PER_CPU(struct llist_head, raised_list); static DEFINE_PER_CPU(struct llist_head, lazy_list); +static bool has_own_interrupt;/* * Claim the entry so that no one else will poke at it.
kernel/irq_work.c
Quoted 10 lines
kernel/irq_work.c +7
-0+void irq_work_tick (void ) +{ +if (!has_own_interrupt) +irq_work_run_list (&__get_cpu_var (raised_list)); +irq_work_run_list (&__get_cpu_var (lazy_list)); +} +/* * Synchronize against the irq_work @entry, ensures the entry is not * currently in use.
Why not something like the below; then it becomes a compile time matter.
---
arch/alpha/include/asm/Kbuild +1 -0
@@ -4,6 +4,7 @@ generic-y += clkdev.h generic-y += cputime.h generic-y += exec.h generic-y += hash.h +generic-y += irq_work.h generic-y += mcs_spinlock.h generic-y += preempt.h generic-y += scatterlist.h
arch/arc/include/asm/Kbuild +1 -0
@@ -18,6 +18,7 @@ generic-y += ioctl.h generic-y += ioctls.h generic-y += ipcbuf.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kmap_types.h generic-y += kvm_para.h generic-y += local.h
arch/arm/include/asm/Kbuild +1 -0
@@ -11,6 +11,7 @@ generic-y += hash.h generic-y += ioctl.h generic-y += ipcbuf.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kdebug.h generic-y += local.h generic-y += local64.h
arch/arm64/include/asm/Kbuild +2 -1
@@ -9,8 +9,8 @@ generic-y += current.h generic-y += delay.h generic-y += div64.h generic-y += dma.h -generic-y += emergency-restart.h generic-y += early_ioremap.h +generic-y += emergency-restart.h generic-y += errno.h generic-y += ftrace.h generic-y += hash.h
@@ -19,6 +19,7 @@ generic-y += ioctl.h generic-y += ioctls.h generic-y += ipcbuf.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kdebug.h generic-y += kmap_types.h generic-y += kvm_para.h
arch/avr32/include/asm/Kbuild +1 -0
@@ -9,6 +9,7 @@ generic-y += exec.h generic-y += futex.h generic-y += hash.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += local.h generic-y += local64.h generic-y += mcs_spinlock.h
arch/blackfin/include/asm/Kbuild +1 -0
@@ -15,6 +15,7 @@ generic-y += hw_irq.h generic-y += ioctl.h generic-y += ipcbuf.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kdebug.h generic-y += kmap_types.h generic-y += kvm_para.h
arch/c6x/include/asm/Kbuild +1 -0
@@ -22,6 +22,7 @@ generic-y += ioctl.h generic-y += ioctls.h generic-y += ipcbuf.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kdebug.h generic-y += kmap_types.h generic-y += local.h
arch/cris/include/asm/Kbuild +1 -0
@@ -8,6 +8,7 @@ generic-y += clkdev.h generic-y += cputime.h generic-y += exec.h generic-y += hash.h +generic-y += irq_work.h generic-y += kvm_para.h generic-y += linkage.h generic-y += mcs_spinlock.h
arch/frv/include/asm/Kbuild +1 -0
@@ -3,6 +3,7 @@ generic-y += clkdev.h generic-y += cputime.h generic-y += exec.h generic-y += hash.h +generic-y += irq_work.h generic-y += mcs_spinlock.h generic-y += preempt.h generic-y += scatterlist.h
arch/hexagon/include/asm/Kbuild +1 -0
@@ -23,6 +23,7 @@ generic-y += ioctls.h generic-y += iomap.h generic-y += ipcbuf.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kdebug.h generic-y += kmap_types.h generic-y += local.h
arch/ia64/include/asm/Kbuild +1 -0
@@ -2,6 +2,7 @@ generic-y += clkdev.h generic-y += exec.h generic-y += hash.h +generic-y += irq_work.h generic-y += kvm_para.h generic-y += mcs_spinlock.h generic-y += preempt.h
arch/m32r/include/asm/Kbuild +1 -0
@@ -3,6 +3,7 @@ generic-y += clkdev.h generic-y += cputime.h generic-y += exec.h generic-y += hash.h +generic-y += irq_work.h generic-y += mcs_spinlock.h generic-y += module.h generic-y += preempt.h
arch/m68k/include/asm/Kbuild +1 -0
@@ -11,6 +11,7 @@ generic-y += hw_irq.h generic-y += ioctl.h generic-y += ipcbuf.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kdebug.h generic-y += kmap_types.h generic-y += kvm_para.h
arch/metag/include/asm/Kbuild +1 -0
@@ -19,6 +19,7 @@ generic-y += ioctl.h generic-y += ioctls.h generic-y += ipcbuf.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kdebug.h generic-y += kmap_types.h generic-y += kvm_para.h
arch/microblaze/include/asm/Kbuild +1 -0
@@ -5,6 +5,7 @@ generic-y += cputime.h generic-y += device.h generic-y += exec.h generic-y += hash.h +generic-y += irq_work.h generic-y += mcs_spinlock.h generic-y += preempt.h generic-y += scatterlist.h
arch/mips/include/asm/Kbuild +1 -0
@@ -3,6 +3,7 @@ generic-y += cputime.h generic-y += current.h generic-y += emergency-restart.h generic-y += hash.h +generic-y += irq_work.h generic-y += local64.h generic-y += mcs_spinlock.h generic-y += mutex.h
arch/mn10300/include/asm/Kbuild +1 -0
@@ -4,6 +4,7 @@ generic-y += clkdev.h generic-y += cputime.h generic-y += exec.h generic-y += hash.h +generic-y += irq_work.h generic-y += mcs_spinlock.h generic-y += preempt.h generic-y += scatterlist.h
arch/openrisc/include/asm/Kbuild +1 -0
@@ -31,6 +31,7 @@ generic-y += ioctl.h generic-y += ioctls.h generic-y += ipcbuf.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kdebug.h generic-y += kmap_types.h generic-y += kvm_para.h
arch/parisc/include/asm/Kbuild +1 -0
@@ -10,6 +10,7 @@ generic-y += exec.h generic-y += hash.h generic-y += hw_irq.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kdebug.h generic-y += kvm_para.h generic-y += local.h
arch/powerpc/include/asm/Kbuild +1 -0
@@ -1,6 +1,7 @@ generic-y += clkdev.h generic-y += hash.h +generic-y += irq_work.h generic-y += mcs_spinlock.h generic-y += preempt.h generic-y += rwsem.h
arch/s390/include/asm/Kbuild +1 -0
@@ -2,6 +2,7 @@ generic-y += clkdev.h generic-y += hash.h +generic-y += irq_work.h generic-y += mcs_spinlock.h generic-y += preempt.h generic-y += scatterlist.h
arch/score/include/asm/Kbuild +1 -0
@@ -6,6 +6,7 @@ generic-y += barrier.h generic-y += clkdev.h generic-y += cputime.h generic-y += hash.h +generic-y += irq_work.h generic-y += mcs_spinlock.h generic-y += preempt.h generic-y += scatterlist.h
arch/sh/include/asm/Kbuild +1 -0
@@ -12,6 +12,7 @@ generic-y += hash.h generic-y += ioctl.h generic-y += ipcbuf.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kvm_para.h generic-y += local.h generic-y += local64.h
arch/sparc/include/asm/Kbuild +1 -0
@@ -8,6 +8,7 @@ generic-y += emergency-restart.h generic-y += exec.h generic-y += hash.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += linkage.h generic-y += local.h generic-y += local64.h
arch/tile/include/asm/Kbuild +1 -0
@@ -17,6 +17,7 @@ generic-y += ioctl.h generic-y += ioctls.h generic-y += ipcbuf.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += local.h generic-y += local64.h generic-y += mcs_spinlock.h
arch/um/include/asm/Kbuild +1 -0
@@ -14,6 +14,7 @@ generic-y += hash.h generic-y += hw_irq.h generic-y += io.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kdebug.h generic-y += mcs_spinlock.h generic-y += mutex.h
arch/unicore32/include/asm/Kbuild +1 -0
@@ -22,6 +22,7 @@ generic-y += ioctl.h generic-y += ioctls.h generic-y += ipcbuf.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kdebug.h generic-y += kmap_types.h generic-y += local.h
arch/x86/include/asm/Kbuild +1 -0
@@ -7,5 +7,6 @@ genhdr-y += unistd_x32.h generic-y += clkdev.h generic-y += cputime.h generic-y += early_ioremap.h +generic-y += irq_work.h generic-y += mcs_spinlock.h generic-y += scatterlist.h
arch/xtensa/include/asm/Kbuild +1 -0
@@ -12,6 +12,7 @@ generic-y += hardirq.h generic-y += hash.h generic-y += ioctl.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kdebug.h generic-y += kmap_types.h generic-y += kvm_para.h
include/asm-generic/irq_work.h +10 -0
@@ -0,0 +1,10 @@ +#ifndef __ASM_IRQ_WORK_H +#define __ASM_IRQ_WORK_H + +static inline bool arch_irq_work_interrupt (void ) +{ +return false; +} + +#undef /* __ASM_IRQ_WORK_H */ +
Re: [RFC PATCH 3/9] irq_work: Force raised irq work to run on irq work interrupt
Quoted 14 lines · Peter Zijlstra
Quoted 12 lines · Frederic Weisbecker
kernel/irq_work.c +1
-0@@ -22,6 +22,7 @@ static DEFINE_PER_CPU(struct llist_head, raised_list); static DEFINE_PER_CPU(struct llist_head, lazy_list); +static bool has_own_interrupt;/* * Claim the entry so that no one else will poke at it.
Quoted 12 lines
kernel/irq_work.c
Quoted 10 lines
kernel/irq_work.c +7
-0+void irq_work_tick (void ) +{ +if (!has_own_interrupt) +irq_work_run_list (&__get_cpu_var (raised_list)); +irq_work_run_list (&__get_cpu_var (lazy_list)); +} +/* * Synchronize against the irq_work @entry, ensures the entry is not * currently in use. Why not something like the below; then it becomes a compile time matter.
Problem is that some archs only have that information at runtime. See x86 for example which tests cpu_has_apic.
Re: [RFC PATCH 3/9] irq_work: Force raised irq work to run on irq work interrupt
Quoted 4 lines · Frederic Weisbecker
Quoted 1 line
Why not something like the below; then it becomes a compile time matter.
Problem is that some archs only have that information at runtime. See x86 for example which tests
cpu_has_apic.
So the other archs can still benefit, no? Also, on x86 cpu features can be used with asm_goto/jump_labels, your approach wrecks that too by copying it into some static variable, see static_cpu_has().
Re: [RFC PATCH 3/9] irq_work: Force raised irq work to run on irq work interrupt
Quoted 9 lines · Peter Zijlstra
Quoted 4 lines · Frederic Weisbecker
Quoted 1 line
Why not something like the below; then it becomes a compile time matter.
Problem is that some archs only have that information at runtime. See x86 for example which tests
cpu_has_apic.So the other archs can still benefit, no? Also, on x86 cpu features can be used with asm_goto/jump_labels, your approach wrecks that too by copying it into some static variable, see
static_cpu_has().
Ah I didn't know about that static_cpu_has().
I'm going to try the way you suggest then.
Thanks.
Re: [RFC PATCH 3/9] irq_work: Force raised irq work to run on irq work interrupt
In reply to Peter ZijlstraQuoted 14 lines · Peter Zijlstra
Quoted 12 lines · Frederic Weisbecker
kernel/irq_work.c +1
-0@@ -22,6 +22,7 @@ static DEFINE_PER_CPU(struct llist_head, raised_list); static DEFINE_PER_CPU(struct llist_head, lazy_list); +static bool has_own_interrupt;/* * Claim the entry so that no one else will poke at it.
Quoted 12 lines
kernel/irq_work.c
Quoted 10 lines
kernel/irq_work.c +7
-0+void irq_work_tick (void ) +{ +if (!has_own_interrupt) +irq_work_run_list (&__get_cpu_var (raised_list)); +irq_work_run_list (&__get_cpu_var (lazy_list)); +} +/* * Synchronize against the irq_work @entry, ensures the entry is not * currently in use. Why not something like the below; then it becomes a compile time matter.
You have a script that does that arch/*/include/asm/Kbuild edit for you right? Is this something in scripts/ ?
Thanks.
Quoted 373 lines
---
arch/alpha/include/asm/Kbuild +1
-0@@ -4,6 +4,7 @@ generic-y += clkdev.h generic-y += cputime.h generic-y += exec.h generic-y += hash.h +generic-y += irq_work.h generic-y += mcs_spinlock.h generic-y += preempt.h generic-y += scatterlist.harch/arc/include/asm/Kbuild +1
-0@@ -18,6 +18,7 @@ generic-y += ioctl.h generic-y += ioctls.h generic-y += ipcbuf.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kmap_types.h generic-y += kvm_para.h generic-y += local.harch/arm/include/asm/Kbuild +1
-0@@ -11,6 +11,7 @@ generic-y += hash.h generic-y += ioctl.h generic-y += ipcbuf.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kdebug.h generic-y += local.h generic-y += local64.harch/arm64/include/asm/Kbuild +2
-1@@ -9,8 +9,8 @@ generic-y += current.h generic-y += delay.h generic-y += div64.h generic-y += dma.h -generic-y += emergency-restart.h generic-y += early_ioremap.h +generic-y += emergency-restart.h generic-y += errno.h generic-y += ftrace.h generic-y += hash.h@@ -19,6 +19,7 @@ generic-y += ioctl.h generic-y += ioctls.h generic-y += ipcbuf.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kdebug.h generic-y += kmap_types.h generic-y += kvm_para.harch/avr32/include/asm/Kbuild +1
-0@@ -9,6 +9,7 @@ generic-y += exec.h generic-y += futex.h generic-y += hash.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += local.h generic-y += local64.h generic-y += mcs_spinlock.harch/blackfin/include/asm/Kbuild +1
-0@@ -15,6 +15,7 @@ generic-y += hw_irq.h generic-y += ioctl.h generic-y += ipcbuf.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kdebug.h generic-y += kmap_types.h generic-y += kvm_para.harch/c6x/include/asm/Kbuild +1
-0@@ -22,6 +22,7 @@ generic-y += ioctl.h generic-y += ioctls.h generic-y += ipcbuf.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kdebug.h generic-y += kmap_types.h generic-y += local.harch/cris/include/asm/Kbuild +1
-0@@ -8,6 +8,7 @@ generic-y += clkdev.h generic-y += cputime.h generic-y += exec.h generic-y += hash.h +generic-y += irq_work.h generic-y += kvm_para.h generic-y += linkage.h generic-y += mcs_spinlock.harch/frv/include/asm/Kbuild +1
-0@@ -3,6 +3,7 @@ generic-y += clkdev.h generic-y += cputime.h generic-y += exec.h generic-y += hash.h +generic-y += irq_work.h generic-y += mcs_spinlock.h generic-y += preempt.h generic-y += scatterlist.harch/hexagon/include/asm/Kbuild +1
-0@@ -23,6 +23,7 @@ generic-y += ioctls.h generic-y += iomap.h generic-y += ipcbuf.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kdebug.h generic-y += kmap_types.h generic-y += local.harch/ia64/include/asm/Kbuild +1
-0@@ -2,6 +2,7 @@ generic-y += clkdev.h generic-y += exec.h generic-y += hash.h +generic-y += irq_work.h generic-y += kvm_para.h generic-y += mcs_spinlock.h generic-y += preempt.harch/m32r/include/asm/Kbuild +1
-0@@ -3,6 +3,7 @@ generic-y += clkdev.h generic-y += cputime.h generic-y += exec.h generic-y += hash.h +generic-y += irq_work.h generic-y += mcs_spinlock.h generic-y += module.h generic-y += preempt.harch/m68k/include/asm/Kbuild +1
-0@@ -11,6 +11,7 @@ generic-y += hw_irq.h generic-y += ioctl.h generic-y += ipcbuf.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kdebug.h generic-y += kmap_types.h generic-y += kvm_para.harch/metag/include/asm/Kbuild +1
-0@@ -19,6 +19,7 @@ generic-y += ioctl.h generic-y += ioctls.h generic-y += ipcbuf.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kdebug.h generic-y += kmap_types.h generic-y += kvm_para.harch/microblaze/include/asm/Kbuild +1
-0@@ -5,6 +5,7 @@ generic-y += cputime.h generic-y += device.h generic-y += exec.h generic-y += hash.h +generic-y += irq_work.h generic-y += mcs_spinlock.h generic-y += preempt.h generic-y += scatterlist.harch/mips/include/asm/Kbuild +1
-0@@ -3,6 +3,7 @@ generic-y += cputime.h generic-y += current.h generic-y += emergency-restart.h generic-y += hash.h +generic-y += irq_work.h generic-y += local64.h generic-y += mcs_spinlock.h generic-y += mutex.harch/mn10300/include/asm/Kbuild +1
-0@@ -4,6 +4,7 @@ generic-y += clkdev.h generic-y += cputime.h generic-y += exec.h generic-y += hash.h +generic-y += irq_work.h generic-y += mcs_spinlock.h generic-y += preempt.h generic-y += scatterlist.harch/openrisc/include/asm/Kbuild +1
-0@@ -31,6 +31,7 @@ generic-y += ioctl.h generic-y += ioctls.h generic-y += ipcbuf.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kdebug.h generic-y += kmap_types.h generic-y += kvm_para.harch/parisc/include/asm/Kbuild +1
-0@@ -10,6 +10,7 @@ generic-y += exec.h generic-y += hash.h generic-y += hw_irq.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kdebug.h generic-y += kvm_para.h generic-y += local.harch/powerpc/include/asm/Kbuild +1
-0@@ -1,6 +1,7 @@ generic-y += clkdev.h generic-y += hash.h +generic-y += irq_work.h generic-y += mcs_spinlock.h generic-y += preempt.h generic-y += rwsem.harch/s390/include/asm/Kbuild +1
-0@@ -2,6 +2,7 @@ generic-y += clkdev.h generic-y += hash.h +generic-y += irq_work.h generic-y += mcs_spinlock.h generic-y += preempt.h generic-y += scatterlist.harch/score/include/asm/Kbuild +1
-0@@ -6,6 +6,7 @@ generic-y += barrier.h generic-y += clkdev.h generic-y += cputime.h generic-y += hash.h +generic-y += irq_work.h generic-y += mcs_spinlock.h generic-y += preempt.h generic-y += scatterlist.harch/sh/include/asm/Kbuild +1
-0@@ -12,6 +12,7 @@ generic-y += hash.h generic-y += ioctl.h generic-y += ipcbuf.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kvm_para.h generic-y += local.h generic-y += local64.harch/sparc/include/asm/Kbuild +1
-0@@ -8,6 +8,7 @@ generic-y += emergency-restart.h generic-y += exec.h generic-y += hash.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += linkage.h generic-y += local.h generic-y += local64.harch/tile/include/asm/Kbuild +1
-0@@ -17,6 +17,7 @@ generic-y += ioctl.h generic-y += ioctls.h generic-y += ipcbuf.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += local.h generic-y += local64.h generic-y += mcs_spinlock.harch/um/include/asm/Kbuild +1
-0@@ -14,6 +14,7 @@ generic-y += hash.h generic-y += hw_irq.h generic-y += io.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kdebug.h generic-y += mcs_spinlock.h generic-y += mutex.harch/unicore32/include/asm/Kbuild +1
-0@@ -22,6 +22,7 @@ generic-y += ioctl.h generic-y += ioctls.h generic-y += ipcbuf.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kdebug.h generic-y += kmap_types.h generic-y += local.harch/x86/include/asm/Kbuild +1
-0@@ -7,5 +7,6 @@ genhdr-y += unistd_x32.h generic-y += clkdev.h generic-y += cputime.h generic-y += early_ioremap.h +generic-y += irq_work.h generic-y += mcs_spinlock.h generic-y += scatterlist.harch/xtensa/include/asm/Kbuild +1
-0@@ -12,6 +12,7 @@ generic-y += hardirq.h generic-y += hash.h generic-y += ioctl.h generic-y += irq_regs.h +generic-y += irq_work.h generic-y += kdebug.h generic-y += kmap_types.h generic-y += kvm_para.hinclude/asm-generic/irq_work.h +10
-0@@ -0,0 +1,10 @@ +#ifndef __ASM_IRQ_WORK_H +#define __ASM_IRQ_WORK_H + +static inline bool arch_irq_work_interrupt (void ) +{ +return false; +} + +#undef /* __ASM_IRQ_WORK_H */ +
Re: [RFC PATCH 3/9] irq_work: Force raised irq work to run on irq work interrupt
Quoted 2 lines · Frederic Weisbecker
You have a script that does that
arch/*/include/asm/Kbuildedit for you right? Is this something inscripts/?
See commit b119fa61d440 ("locking/mcs: Order the header files in Kbuild of each architecture in alphabetical order")
That script is a little buggy, but should work in the current case. So first do something like:
for i in arch/*/include/asm/Kbuild; do echo "generic-y += foo.h" >> $i; done
Then run that script.
IIRC the bug was where a single line contained multiple headers and/or a continuation. I should have a better version somewhere, but this one was easier to find.
Re: [RFC PATCH 3/9] irq_work: Force raised irq work to run on irq work interrupt
Quoted 14 lines · Peter Zijlstra
Quoted 2 lines · Frederic Weisbecker
You have a script that does that
arch/*/include/asm/Kbuildedit for you right? Is this something inscripts/?See commit
b119fa61d440("locking/mcs: Order the header files inKbuildof each architecture in alphabetical order")That script is a little buggy, but should work in the current case. So first do something like:
for i in arch/*/include/asm/Kbuild; do echo "generic-y += foo.h" >> $i; done
Quoted 5 lines
Then run that script.
IIRC the bug was where a single line contained multiple headers and/or a continuation. I should have a better version somewhere, but this one was easier to find.
I see. Meanwhile I've applied your suggestion as a separate patch in the series. May I use your SOB?
Thanks!
Re: [RFC PATCH 3/9] irq_work: Force raised irq work to run on irq work interrupt
Quoted 15 lines · Frederic Weisbecker
Quoted 14 lines · Peter Zijlstra
Quoted 2 lines · Frederic Weisbecker
You have a script that does that
arch/*/include/asm/Kbuildedit for you right? Is this something inscripts/?See commit
b119fa61d440("locking/mcs: Order the header files inKbuildof each architecture in alphabetical order")That script is a little buggy, but should work in the current case. So first do something like:
for i in arch/*/include/asm/Kbuild; do echo "generic-y += foo.h" >> $i; done
Quoted 8 lines
Quoted 5 lines
Then run that script.
IIRC the bug was where a single line contained multiple headers and/or a continuation. I should have a better version somewhere, but this one was easier to find.
I see. Meanwhile I've applied your suggestion as a separate patch in the series. May I use your SOB?
Sure.
RFC PATCH 4/9 x86: Build irq work only if local apic support
In reply to Frederic WeisbeckerIrq work won't run without local apic support. This check is currently hacked-up through ifdeffery. Lets do conditional build instead. That way we avoid further ifdefs with the implementation of arch_irq_work_has_own_interrupt().
--- arch/x86/Kconfig | 4 ++++ arch/x86/include/asm/entry_arch.h | 2 +- arch/x86/kernel/Makefile | 2 +- arch/x86/kernel/entry_64.S | 2 +- arch/x86/kernel/irq_work.c | 2 -- arch/x86/kernel/irqinit.c | 2 +- 6 files changed, 8 insertions(+), 6 deletions(-)
arch/x86/Kconfig +4 -0
@@ -837,6 +837,10 @@ config X86_LOCAL_APIC def_bool y depends on X86_64 || SMP || X86_32_NON_STANDARD || X86_UP_APIC || PCI_MSI +config X86_IRQ_WORK + def_bool y + depends on X86_LOCAL_APIC && IRQ_WORK + config X86_IO_APIC def_bool y depends on X86_64 || SMP || X86_32_NON_STANDARD || X86_UP_IOAPIC || PCI_MSI
arch/x86/include/asm/entry_arch.h +1 -1
@@ -38,7 +38,7 @@ BUILD_INTERRUPT(apic_timer_interrupt,LOCAL_TIMER_VECTOR) BUILD_INTERRUPT(error_interrupt,ERROR_APIC_VECTOR) BUILD_INTERRUPT(spurious_interrupt,SPURIOUS_APIC_VECTOR) -#ifdef CONFIG_IRQ_WORK +#ifdef CONFIG_X86_IRQ_WORK BUILD_INTERRUPT(irq_work_interrupt, IRQ_WORK_VECTOR)#endif
arch/x86/kernel/Makefile +1 -1
@@ -22,7 +22,7 @@ obj-y := process_$(BITS).o signal.o entry_$(BITS).o obj-y += traps.o irq.o irq_$(BITS).o dumpstack_$(BITS).o obj-y += time.o ioport.o ldt.o dumpstack.o nmi.o obj-y += setup.o x86_init.o i8259.o irqinit.o jump_label.o -obj-$(CONFIG_IRQ_WORK) += irq_work.o +obj-$(CONFIG_X86_IRQ_WORK) += irq_work.o obj-y += probe_roms.o obj-$(CONFIG_X86_32) += i386_ksyms_32.o obj-$(CONFIG_X86_64) += sys_x86_64.o x8664_ksyms_64.o
arch/x86/kernel/entry_64.S +1 -1
@@ -1043,7 +1043,7 @@ apicinterrupt ERROR_APIC_VECTOR \ apicinterrupt SPURIOUS_APIC_VECTOR \ spurious_interrupt smp_spurious_interrupt -#ifdef CONFIG_IRQ_WORK +#ifdef CONFIG_X86_IRQ_WORK apicinterrupt IRQ_WORK_VECTOR \ irq_work_interrupt smp_irq_work_interrupt #endif
arch/x86/kernel/irq_work.c +0 -2
@@ -40,11 +40,9 @@ __visible void smp_trace_irq_work_interrupt(struct pt_regs *regs)void arch_irq_work_raise (void ) { -#ifdef CONFIG_X86_LOCAL_APIC if (!cpu_has_apic)return ; apic->send_IPI_self (IRQ_WORK_VECTOR );apic_wait_icr_idle (); -#endif }
arch/x86/kernel/irqinit.c +1 -1
@@ -176,7 +176,7 @@ static void __init apic_intr_init(void)alloc_intr_gate (ERROR_APIC_VECTOR , error_interrupt);/* IRQ work interrupts: */ -# ifdef CONFIG_IRQ_WORK +# ifdef CONFIG_X86_IRQ_WORK alloc_intr_gate (IRQ_WORK_VECTOR , irq_work_interrupt);# endif
Signature · 1 line
1.8.3.1
RFC PATCH 5/9 x86: Tell irq work about self IPI support
In reply to Frederic Weisbeckerx86 supports irq work when local apic is available. This is partly known on runtime so lets implement arch_irq_work_has_own_interrupt() accordingly.
This should be safely called after setup_arch().
--- arch/x86/kernel/irq_work.c | 5 +++++ 1 file changed, 5 insertions(+)
arch/x86/kernel/irq_work.c +5 -0
@@ -38,6 +38,11 @@ __visible void smp_trace_irq_work_interrupt(struct pt_regs *regs)exiting_irq (); } +int __init arch_irq_work_has_own_interrupt (void ) +{ +return cpu_has_apic; +} +void arch_irq_work_raise (void ) {if (!cpu_has_apic)
Signature · 1 line
1.8.3.1
RFC PATCH 6/9 arm: Tell irq work about self IPI support
In reply to Frederic WeisbeckerARM irq work IPI support depends on SMP support. That information is partly known at early boottime. Lets implement arch_irq_work_has_own_interrupt() accordingly.
--- arch/arm/kernel/smp.c | 5 +++++ 1 file changed, 5 insertions(+)
arch/arm/kernel/smp.c +5 -0
@@ -506,6 +506,11 @@ void arch_irq_work_raise(void)if (is_smp ()) smp_cross_call(cpumask_of(smp_processor_id()), IPI_IRQ_WORK); } + +int __init arch_irq_work_has_own_interrupt (void ) +{ +return is_smp (); +}#endif #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
Signature · 1 line
1.8.3.1
Re: [RFC PATCH 6/9] arm: Tell irq work about self IPI support
Quoted 3 lines · Frederic Weisbecker
ARM irq work IPI support depends on SMP support. That information is partly known at early boottime. Lets implement
arch_irq_work_has_own_interrupt()accordingly.
Sorry, this makes no sense on its own. As there's no context for this to know whether it's an appropriate change or not, it would be irresponsible fo me to provide an ack for something I don't understand.
Signature · 2 lines
FTTC broadband for 0.8mile line: currently at 9.5Mbps down 400kbps up according to speedtest.net.
Re: [RFC PATCH 6/9] arm: Tell irq work about self IPI support
Quoted 8 lines · Russell King - ARM Linux
Quoted 3 lines · Frederic Weisbecker
ARM irq work IPI support depends on SMP support. That information is partly known at early boottime. Lets implement
arch_irq_work_has_own_interrupt()accordingly.Sorry, this makes no sense on its own. As there's no context for this to know whether it's an appropriate change or not, it would be irresponsible fo me to provide an ack for something I don't understand.
Ah you're right, I should have Cc'ed you for all patches in the series. I didn't because I wanted to avoid spaming you but sending that patch alone doesn't make much sense.
Let me bounce the rest to you.
RFC PATCH 7/9 arm64: Tell irq work about self IPI support
In reply to Frederic WeisbeckerARM64 irq work support depends on __smp_cross_call to point to some relevant IRQ controller operations. This information should be available after the call to init_IRQ().
Lets implement arch_irq_work_has_own_interrupt() accordingly.
--- arch/arm64/kernel/smp.c | 5 +++++ 1 file changed, 5 insertions(+)
arch/arm64/kernel/smp.c +5 -0
@@ -534,6 +534,11 @@ void arch_irq_work_raise(void)if (__smp_cross_call) smp_cross_call(cpumask_of(smp_processor_id()), IPI_IRQ_WORK); } + +int __init arch_irq_work_has_own_interrupt (void ) +{ +return !!__smp_cross_call; +}#endif static DEFINE_RAW_SPINLOCK (stop_lock);
Signature · 1 line
1.8.3.1
RFC PATCH 8/9 nohz: Consolidate nohz full init code
In reply to Frederic WeisbeckerThe supports for CONFIG_NO_HZ_FULL_ALL=y and the nohz_full= kernel parameter both have their own way to do the same thing: allocate full dynticks cpumasks, fill them and initialize some state variables.
Lets consolidate that all in the same place.
While at it, convert some regular printk message to warnings when fundamental allocations fail.
--- kernel/time/tick-sched.c | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-)
kernel/time/tick-sched.c +19 -19
@@ -295,22 +295,12 @@ out:/* Parse the boot-time nohz CPU list from the kernel parameters. */ static int __init tick_nohz_full_setup (char *str) { -int cpu; -alloc_bootmem_cpumask_var (&tick_nohz_full_mask); -alloc_bootmem_cpumask_var (&housekeeping_mask);if (cpulist_parse (str, tick_nohz_full_mask) <0 ) {pr_warning ("NOHZ: Incorrect nohz_full cpumask\n" ); +free_bootmem_cpumask_var (tick_nohz_full_mask);return 1 ; } - - cpu =smp_processor_id (); -if (cpumask_test_cpu (cpu, tick_nohz_full_mask)) { -pr_warning ("NO_HZ: Clearing %d from nohz_full range for timekeeping\n" , cpu); -cpumask_clear_cpu (cpu, tick_nohz_full_mask); - } -cpumask_andnot (housekeeping_mask, - cpu_possible_mask, tick_nohz_full_mask); tick_nohz_full_running = true;return 1 ;
@@ -349,18 +339,11 @@ static int tick_nohz_init_all(void)#ifdef CONFIG_NO_HZ_FULL_ALL if (!alloc_cpumask_var (&tick_nohz_full_mask,GFP_KERNEL )) { -pr_err ("NO_HZ: Can't allocate full dynticks cpumask\n" ); -return err; - } -if (!alloc_cpumask_var (&housekeeping_mask,GFP_KERNEL )) { -pr_err ("NO_HZ: Can't allocate not-full dynticks cpumask\n" ); +WARN (1 ,"NO_HZ: Can't allocate full dynticks cpumask\n" );return err; } err =0 ;cpumask_setall (tick_nohz_full_mask); -cpumask_clear_cpu (smp_processor_id (), tick_nohz_full_mask); -cpumask_clear (housekeeping_mask); -cpumask_set_cpu (smp_processor_id (), housekeeping_mask); tick_nohz_full_running = true;#endif return err;
@@ -375,6 +358,23 @@ void __init tick_nohz_init(void)return ; } +if (!alloc_cpumask_var (&housekeeping_mask,GFP_KERNEL )) { +WARN (1 ,"NO_HZ: Can't allocate not-full dynticks cpumask\n" ); +cpumask_clear (tick_nohz_full_mask); + tick_nohz_full_running = false; +return ; + } + + cpu =smp_processor_id (); + +if (cpumask_test_cpu (cpu, tick_nohz_full_mask)) { +pr_warning ("NO_HZ: Clearing %d from nohz_full range for timekeeping\n" , cpu); +cpumask_clear_cpu (cpu, tick_nohz_full_mask); + } + +cpumask_andnot (housekeeping_mask, + cpu_possible_mask, tick_nohz_full_mask); +for_each_cpu (cpu, tick_nohz_full_mask)context_tracking_cpu_set (cpu);
Signature · 1 line
1.8.3.1
RFC PATCH 9/9 nohz: nohz full depends on irq work self IPI support
In reply to Frederic WeisbeckerThe nohz full functionality depends on IRQ work to trigger its own interrupts. As it's used to restart the tick, we can't rely on the tick fallback for irq work callbacks, ie: we can't use the tick to restart the tick itself as it may well be completely stopped.
Lets reject the full dynticks initialization if that arch support isn't available.
As a side effect, this makes sure that nohz kick is never called from the tick. Some lockup have been reported due to the kick trying to cancel the tick hrtimer from the tick itself, which of course doesn't work.
--- include/linux/irq_work.h | 1 + kernel/time/tick-sched.c | 14 ++++++++++++++ 2 files changed, 15 insertions(+)
include/linux/irq_work.h +1 -0
@@ -33,6 +33,7 @@ void init_irq_work(struct irq_work *work, void (*func)(struct irq_work *))#define DEFINE_IRQ_WORK (name, _f) struct irq_work name = { .func = (_f), }bool irq_work_queue (struct irq_work *work); +int arch_irq_work_has_own_interrupt (void );#ifdef CONFIG_SMP bool irq_work_queue_on (struct irq_work *work,int cpu);
kernel/time/tick-sched.c +14 -0
@@ -365,6 +365,20 @@ void __init tick_nohz_init(void)return ; } +/* +* Full dynticks uses irq work to drive the tick rescheduling on safe +* locking contexts. But then we need irq work to raise its own +* interrupts to avoid circular dependency on the tick +*/ +if (!arch_irq_work_has_own_interrupt ()) { +pr_warning ("NO_HZ: Can't run full dynticks because arch doesn't " +"support irq work self-IPIs\n" ); +cpumask_clear (tick_nohz_full_mask); +cpumask_copy (housekeeping_mask, cpu_possible_mask); + tick_nohz_full_running = false; +return ; + } + cpu =smp_processor_id ();if (cpumask_test_cpu (cpu, tick_nohz_full_mask)) {
Signature · 1 line
1.8.3.1