timer: make deferrable cpu unbound timers really not bound to a cpu
v2 cover timer: make deferrable cpu unbound timers really not bound to a cpu
These patches introduce support for global deferrable timers and this feature is very useful for the mobile chipsets and initial discussion link -[1]. Lastest discussion was with - [2] as these patches are in downstream kernel for quite a long time.
[1]- https://lore.kernel.org/patchwork/patch/500541/
[2]- https://lkml.org/lkml/2020/3/16/147
Changelog:
v1 -> v2:
- Fixed tglx comments about warning.
- Thanks Pavan for your suggestion.
Joonwoo Park (1):
timer: make deferrable cpu unbound timers really not bound to a cpu
Prasad Sodagudi (1):
sched: Add a check for cpu unbound deferrable timers
include/linux/timer.h | 3 +++ kernel/time/tick-sched.c | 8 +++++- kernel/time/timer.c | 63 ++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 68 insertions(+), 6 deletions(-)
Signature · 2 lines
The Qualcomm Innovation Center, Inc. is a member of the Code Aurora Forum, a Linux Foundation Collaborative Project
PATCH v2 1/2 timer: make deferrable cpu unbound timers really not bound to a cpu
From: Joonwoo Park <[email protected]>
When a deferrable work (INIT_DEFERRABLE_WORK, etc.) is queued via queue_delayed_work() it's probably intended to run the work item on any CPU that isn't idle. However, we queue the work to run at a later time by starting a deferrable timer that binds to whatever CPU the work is queued on which is same with queue_delayed_work_on(smp_processor_id()) effectively.
As a result WORK_CPU_UNBOUND work items aren't really cpu unbound now. In fact this is perfectly fine with UP kernel and also won't affect much a system without dyntick with SMP kernel too as every cpus run timers periodically. But on SMP systems with dyntick current implementation leads deferrable timers not very scalable because the timer's base which has queued the deferrable timer won't wake up till next non-deferrable timer expires even though there are possible other non idle cpus are running which are able to run expired deferrable timers.
The deferrable work is a good example of the current implementation's victim like below.
INIT_DEFERRABLE_WORK(&dwork, fn);
CPU 0 CPU 1
queue_delayed_work(wq, &dwork, HZ);
queue_delayed_work_on(WORK_CPU_UNBOUND);
...
__mod_timer() -> queues timer to the
current cpu's timer
base.
...tick_nohz_idle_enter() -> cpu enters idle.
A second later
cpu 0 is now in idle. cpu 1 exits idle or wasn't in idle so
now it's in active but won't
cpu 0 won't wake up till next handle cpu unbound deferrable timer
non-deferrable timer expires. as it's in cpu 0's timer base.
To make all cpu unbound deferrable timers are scalable, introduce a common timer base which is only for cpu unbound deferrable timers to make those are indeed cpu unbound so that can be scheduled by any of non idle cpus. This common timer fixes scalability issue of delayed work and all other cpu unbound deferrable timer using implementations.
--- kernel/time/timer.c | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-)
kernel/time/timer.c +31 -5
@@ -220,6 +220,7 @@ static void timer_update_keys(struct work_struct *work); static DECLARE_WORK(timer_update_work, timer_update_keys);#ifdef CONFIG_SMP +struct timer_base timer_base_deferrable;unsigned int sysctl_timer_migration =1 ;DEFINE_STATIC_KEY_FALSE (timers_migration_enabled);
@@ -841,8 +842,14 @@ static inline struct timer_base *get_timer_cpu_base(u32 tflags, u32 cpu)* If the timer is deferrable and NO_HZ_COMMON is set then we need * to use the deferrable base. */ -if (IS_ENABLED (CONFIG_NO_HZ_COMMON ) && (tflags &TIMER_DEFERRABLE )) - base =per_cpu_ptr (&timer_bases[BASE_DEF ], cpu); +if (IS_ENABLED (CONFIG_NO_HZ_COMMON ) && (tflags &TIMER_DEFERRABLE )) { +#ifdef CONFIG_SMP + base = &timer_base_deferrable; +#endif +if (tflags &TIMER_PINNED ) + base =per_cpu_ptr (&timer_bases[BASE_DEF ], cpu); + } +return base; }
@@ -854,8 +861,14 @@ static inline struct timer_base *get_timer_this_cpu_base(u32 tflags)* If the timer is deferrable and NO_HZ_COMMON is set then we need * to use the deferrable base. */ -if (IS_ENABLED (CONFIG_NO_HZ_COMMON ) && (tflags &TIMER_DEFERRABLE )) - base =this_cpu_ptr (&timer_bases[BASE_DEF ]); +if (IS_ENABLED (CONFIG_NO_HZ_COMMON ) && (tflags &TIMER_DEFERRABLE )) { +#ifdef CONFIG_SMP + base = &timer_base_deferrable; +#endif +if (tflags &TIMER_PINNED ) + base =this_cpu_ptr (&timer_bases[BASE_DEF ]); + } +return base; }
@@ -1785,8 +1798,12 @@ static __latent_entropy void run_timer_softirq(struct softirq_action *h)struct timer_base *base =this_cpu_ptr (&timer_bases[BASE_STD ]);__run_timers (base); -if (IS_ENABLED (CONFIG_NO_HZ_COMMON )) +if (IS_ENABLED (CONFIG_NO_HZ_COMMON )) {__run_timers (this_cpu_ptr (&timer_bases[BASE_DEF ])); +if (tick_do_timer_cpu ==TICK_DO_TIMER_NONE || + tick_do_timer_cpu ==smp_processor_id ()) +__run_timers (&timer_base_deferrable); + } }/*
@@ -2025,6 +2042,14 @@ static void __init init_timer_cpu(int cpu) } } +static void __init init_timer_deferrable_global (void ) +{ + timer_base_deferrable.cpu = nr_cpu_ids; +raw_spin_lock_init (&timer_base_deferrable.lock); + timer_base_deferrable.clk = jiffies; +timer_base_init_expiry_lock (&timer_base_deferrable); +} +static void __init init_timer_cpus (void ) {int cpu;
@@ -2036,6 +2061,7 @@ static void __init init_timer_cpus(void)void __init init_timers (void ) {init_timer_cpus (); +init_timer_deferrable_global ();open_softirq (TIMER_SOFTIRQ , run_timer_softirq); }
Signature · 2 lines
The Qualcomm Innovation Center, Inc. is a member of the Code Aurora Forum, a Linux Foundation Collaborative Project
kbuild test robot
build botThank you for the patch! Yet something to improve:
Hi Prasad,
Thank you for the patch! Yet something to improve:
[auto build test ERROR on tip/timers/core]
[also build test ERROR on tip/auto-latest tip/timers/nohz v5.7-rc3 next-20200428] [if your patch is applied to the wrong git tree, please drop us a note to help improve the system. BTW, we also suggest to use '--base' option to specify the base tree in git format-patch, please see https://stackoverflow.com/a/37406982]
url: https://github.com/0day-ci/linux/commits/Prasad-Sodagudi/timer-make-deferrable-cpu-unbound-timers-really-not-bound-to-a-cpu/20200429-060558 base: https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git 4479730e9263befbb9ce9563a09563db2acb8f7c config: i386-tinyconfig (attached as .config) compiler: gcc-7 (Ubuntu 7.5.0-6ubuntu2) 7.5.0 reproduce: # save the attached .config to linux build tree make ARCH=i386
If you fix the issue, kindly add following tag as appropriate
All errors (new ones prefixed by >>):
kernel/time/timer.c: In function 'run_timer_softirq':
>> kernel/time/timer.c:1804:18: error: 'timer_base_deferrable' undeclared (first use in this function); did you mean 'timer_base_lock_expiry'?
__run_timers(&timer_base_deferrable);
^~~~~~~~~~~~~~~~~~~~~
timer_base_lock_expiry
kernel/time/timer.c:1804:18: note: each undeclared identifier is reported only once for each function it appears in
kernel/time/timer.c: In function 'init_timer_deferrable_global':
kernel/time/timer.c:2046:2: error: 'timer_base_deferrable' undeclared (first use in this function); did you mean 'timer_base_lock_expiry'?
timer_base_deferrable.cpu = nr_cpu_ids;
^~~~~~~~~~~~~~~~~~~~~
timer_base_lock_expiryvim +1804 kernel/time/timer.c
1791
1792 /*
1793 * This function runs timers and the timer-tq in bottom half context.
1794 */
1795 static __latent_entropy void run_timer_softirq(struct softirq_action *h)
1796 {
1797 struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]);
1798
1799 __run_timers(base);
1800 if (IS_ENABLED(CONFIG_NO_HZ_COMMON)) {
1801 __run_timers(this_cpu_ptr(&timer_bases[BASE_DEF]));
1802 if (tick_do_timer_cpu == TICK_DO_TIMER_NONE ||
1803 tick_do_timer_cpu == smp_processor_id())
> 1804 __run_timers(&timer_base_deferrable);
1805 }
1806 }
1807
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org
PATCH v2 2/2 sched: Add a check for cpu unbound deferrable timers
In reply to Prasad SodagudiAdd a check for cpu unbound deferrable timer expiry and raise softirq for handling the expired timers so that the CPU can process the cpu unbound deferrable times as early as possible when a cpu tries to enter/exit idle loop.
--- include/linux/timer.h | 3 +++ kernel/time/tick-sched.c | 8 +++++++- kernel/time/timer.c | 29 ++++++++++++++++++++++++++++- 3 files changed, 38 insertions(+), 2 deletions(-)
include/linux/timer.h +3 -0
@@ -172,6 +172,9 @@ extern int del_timer(struct timer_list * timer);extern int mod_timer (struct timer_list *timer,unsigned long expires);extern int mod_timer_pending (struct timer_list *timer,unsigned long expires);extern int timer_reduce (struct timer_list *timer,unsigned long expires); +#ifdef CONFIG_SMP +extern bool check_pending_deferrable_timers (int cpu); +#endif /* * The jiffies value which is added to now, when there is no timer
kernel/time/tick-sched.c +7 -1
@@ -23,6 +23,7 @@#include <linux/module.h> #include <linux/irq_work.h> #include <linux/posix-timers.h> +#include <linux/timer.h> #include <linux/context_tracking.h> #include <linux/mm.h>
@@ -1274,8 +1275,13 @@ static inline void tick_nohz_irq_enter(void) now =ktime_get ();if (ts->idle_active)tick_nohz_stop_idle (ts, now); -if (ts->tick_stopped) +if (ts->tick_stopped) {tick_nohz_update_jiffies (now); +#ifdef CONFIG_SMP +if (check_pending_deferrable_timers (smp_processor_id ())) +raise_softirq_irqoff (TIMER_SOFTIRQ ); +#endif + } }#else
kernel/time/timer.c +28 -1
@@ -221,6 +221,7 @@ static DECLARE_WORK(timer_update_work, timer_update_keys);#ifdef CONFIG_SMP struct timer_base timer_base_deferrable; +static atomic_t deferrable_pending;unsigned int sysctl_timer_migration =1 ;DEFINE_STATIC_KEY_FALSE (timers_migration_enabled);
@@ -1610,6 +1611,31 @@ static u64 cmp_next_hrtimer_event(u64 basem, u64 expires)return DIV_ROUND_UP_ULL (nextevt,TICK_NSEC ) *TICK_NSEC ; } + +#ifdef CONFIG_SMP +/* +* check_pending_deferrable_timers - Check for unbound deferrable timer expiry +* @cpu - Current CPU +* +* The function checks whether any global deferrable pending timers +* are exipired or not. This function does not check cpu bounded +* diferrable pending timers expiry. +* +* The function returns true when a cpu unbounded deferrable timer is expired. +*/ +bool check_pending_deferrable_timers (int cpu) +{ +if (cpu == tick_do_timer_cpu || + tick_do_timer_cpu ==TICK_DO_TIMER_NONE ) { +if (time_after_eq (jiffies, timer_base_deferrable.clk) + && !atomic_cmpxchg (&deferrable_pending,0 ,1 )) { +return true; + } + } +return false; +} +#endif +/** * get_next_timer_interrupt - return the time (clock mono) of the next timer * @basej: base time jiffies
@@ -1800,7 +1826,8 @@ static __latent_entropy void run_timer_softirq(struct softirq_action *h)__run_timers (base);if (IS_ENABLED (CONFIG_NO_HZ_COMMON )) {__run_timers (this_cpu_ptr (&timer_bases[BASE_DEF ])); -if (tick_do_timer_cpu ==TICK_DO_TIMER_NONE || +if ((atomic_cmpxchg (&deferrable_pending,1 ,0 ) && + tick_do_timer_cpu ==TICK_DO_TIMER_NONE ) || tick_do_timer_cpu ==smp_processor_id ())__run_timers (&timer_base_deferrable); }
Signature · 2 lines
The Qualcomm Innovation Center, Inc. is a member of the Code Aurora Forum, a Linux Foundation Collaborative Project
kbuild test robot
build botRe: [PATCH v2 2/2] sched: Add a check for cpu unbound deferrable timers
Thank you for the patch! Perhaps something to improve:
Hi Prasad,
Thank you for the patch! Perhaps something to improve:
[auto build test WARNING on tip/timers/core]
[also build test WARNING on tip/auto-latest tip/timers/nohz v5.7-rc3 next-20200428] [if your patch is applied to the wrong git tree, please drop us a note to help improve the system. BTW, we also suggest to use '--base' option to specify the base tree in git format-patch, please see https://stackoverflow.com/a/37406982]
url: https://github.com/0day-ci/linux/commits/Prasad-Sodagudi/timer-make-deferrable-cpu-unbound-timers-really-not-bound-to-a-cpu/20200429-060558 base: https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git 4479730e9263befbb9ce9563a09563db2acb8f7c config: nds32-defconfig (attached as .config) compiler: nds32le-linux-gcc (GCC) 9.3.0
reproduce:
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day GCC_VERSION=9.3.0 make.cross ARCH=nds32
If you fix the issue, kindly add following tag as appropriate
All warnings (new ones prefixed by >>):
In file included from ./arch/nds32/include/generated/asm/cmpxchg.h:1,
from include/asm-generic/atomic.h:12,
from ./arch/nds32/include/generated/asm/atomic.h:1,
from include/linux/atomic.h:7,
from include/asm-generic/bitops/lock.h:5,
from include/asm-generic/bitops.h:31,
from ./arch/nds32/include/generated/asm/bitops.h:1,
from include/linux/bitops.h:29,
from include/linux/kernel.h:12,
from include/linux/list.h:9,
from include/linux/smp.h:12,
from include/linux/kernel_stat.h:5,
from kernel/time/timer.c:21:
kernel/time/timer.c: In function 'run_timer_softirq':
kernel/time/timer.c:1828:24: error: 'deferrable_pending' undeclared (first use in this function)
1828 | if ((atomic_cmpxchg(&deferrable_pending, 1, 0) &&
| ^~~~~~~~~~~~~~~~~~
include/asm-generic/cmpxchg.h:97:17: note: in definition of macro 'cmpxchg_local'
97 | ((__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o),\
| ^~~
>> include/asm-generic/atomic.h:196:38: note: in expansion of macro 'cmpxchg'
196 | #define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), (old), (new)))
| ^~~~~~~
>> kernel/time/timer.c:1828:8: note: in expansion of macro 'atomic_cmpxchg'
1828 | if ((atomic_cmpxchg(&deferrable_pending, 1, 0) &&
| ^~~~~~~~~~~~~~
kernel/time/timer.c:1828:24: note: each undeclared identifier is reported only once for each function it appears in
1828 | if ((atomic_cmpxchg(&deferrable_pending, 1, 0) &&
| ^~~~~~~~~~~~~~~~~~
include/asm-generic/cmpxchg.h:97:17: note: in definition of macro 'cmpxchg_local'
97 | ((__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o),\
| ^~~
>> include/asm-generic/atomic.h:196:38: note: in expansion of macro 'cmpxchg'
196 | #define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), (old), (new)))
| ^~~~~~~
>> kernel/time/timer.c:1828:8: note: in expansion of macro 'atomic_cmpxchg'
1828 | if ((atomic_cmpxchg(&deferrable_pending, 1, 0) &&
| ^~~~~~~~~~~~~~
kernel/time/timer.c:1831:18: error: 'timer_base_deferrable' undeclared (first use in this function)
1831 | __run_timers(&timer_base_deferrable);
| ^~~~~~~~~~~~~~~~~~~~~
kernel/time/timer.c: In function 'init_timer_deferrable_global':
kernel/time/timer.c:2073:2: error: 'timer_base_deferrable' undeclared (first use in this function)
2073 | timer_base_deferrable.cpu = nr_cpu_ids;
| ^~~~~~~~~~~~~~~~~~~~~vim +/atomic_cmpxchg +1828 kernel/time/timer.c
1817
1818 /*
1819 * This function runs timers and the timer-tq in bottom half context.
1820 */
1821 static __latent_entropy void run_timer_softirq(struct softirq_action *h)
1822 {
1823 struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]);
1824
1825 __run_timers(base);
1826 if (IS_ENABLED(CONFIG_NO_HZ_COMMON)) {
1827 __run_timers(this_cpu_ptr(&timer_bases[BASE_DEF]));
> 1828 if ((atomic_cmpxchg(&deferrable_pending, 1, 0) &&
1829 tick_do_timer_cpu == TICK_DO_TIMER_NONE) ||
1830 tick_do_timer_cpu == smp_processor_id())
1831 __run_timers(&timer_base_deferrable);
1832 }
1833 }
1834
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org
kbuild test robot
build botRe: [PATCH v2 2/2] sched: Add a check for cpu unbound deferrable timers
In reply to Prasad SodagudiThank you for the patch! Yet something to improve:
Hi Prasad,
Thank you for the patch! Yet something to improve:
[auto build test ERROR on tip/timers/core]
[also build test ERROR on tip/auto-latest tip/timers/nohz v5.7-rc3 next-20200428] [if your patch is applied to the wrong git tree, please drop us a note to help improve the system. BTW, we also suggest to use '--base' option to specify the base tree in git format-patch, please see https://stackoverflow.com/a/37406982]
url: https://github.com/0day-ci/linux/commits/Prasad-Sodagudi/timer-make-deferrable-cpu-unbound-timers-really-not-bound-to-a-cpu/20200429-060558 base: https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git 4479730e9263befbb9ce9563a09563db2acb8f7c config: i386-tinyconfig (attached as .config) compiler: gcc-7 (Ubuntu 7.5.0-6ubuntu2) 7.5.0 reproduce: # save the attached .config to linux build tree make ARCH=i386
If you fix the issue, kindly add following tag as appropriate
All errors (new ones prefixed by >>):
kernel/time/timer.c: In function 'run_timer_softirq':
>> kernel/time/timer.c:1828:24: error: 'deferrable_pending' undeclared (first use in this function); did you mean 'detach_if_pending'?
if ((atomic_cmpxchg(&deferrable_pending, 1, 0) &&
^~~~~~~~~~~~~~~~~~
detach_if_pending
kernel/time/timer.c:1828:24: note: each undeclared identifier is reported only once for each function it appears in
kernel/time/timer.c:1831:18: error: 'timer_base_deferrable' undeclared (first use in this function); did you mean 'timer_base_lock_expiry'?
__run_timers(&timer_base_deferrable);
^~~~~~~~~~~~~~~~~~~~~
timer_base_lock_expiry
kernel/time/timer.c: In function 'init_timer_deferrable_global':
kernel/time/timer.c:2073:2: error: 'timer_base_deferrable' undeclared (first use in this function); did you mean 'timer_base_lock_expiry'?
timer_base_deferrable.cpu = nr_cpu_ids;
^~~~~~~~~~~~~~~~~~~~~
timer_base_lock_expiryvim +1828 kernel/time/timer.c
1817
1818 /*
1819 * This function runs timers and the timer-tq in bottom half context.
1820 */
1821 static __latent_entropy void run_timer_softirq(struct softirq_action *h)
1822 {
1823 struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]);
1824
1825 __run_timers(base);
1826 if (IS_ENABLED(CONFIG_NO_HZ_COMMON)) {
1827 __run_timers(this_cpu_ptr(&timer_bases[BASE_DEF]));
> 1828 if ((atomic_cmpxchg(&deferrable_pending, 1, 0) &&
1829 tick_do_timer_cpu == TICK_DO_TIMER_NONE) ||
1830 tick_do_timer_cpu == smp_processor_id())
1831 __run_timers(&timer_base_deferrable);
1832 }
1833 }
1834
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org