cpuidle: Replace use_deepest_state flag by forced_idle_latency_limit_ns
Subsystems (3)
PATCH v2 1/2 cpuidle: Replace use_deepest_state flag by forced_idle_latency_limit_ns
We want to specify a latency constraint when choosing an idle state at play_idle time. Instead of duplicating the information in the structure or propagate the latency in the call stack, change the use_deepest_state by forced_latency_limit_ns to introduce this constraint. The idea being that when it is set, idle is forced (i.e. no governors), but there is a latency limit for the state to use.
A zero latency constraint means "do not use the deepest idle state path" as the 'use_deepest_state' boolean was used in the cpuidle_idle_call.
In addition add the play_idle_precise() function getting a nanosec base latency constraint as parameter. The play_idle() function becomes a wrapper passing an U64_MAX to play_idle_precise().
--- drivers/cpuidle/cpuidle.c | 10 +++++----- include/linux/cpu.h | 6 +++++- include/linux/cpuidle.h | 6 +++--- kernel/sched/idle.c | 14 +++++++------- 4 files changed, 20 insertions(+), 16 deletions(-)
drivers/cpuidle/cpuidle.c +5 -5
@@ -100,19 +100,19 @@ static int find_deepest_state(struct cpuidle_driver *drv,/** * cpuidle_use_deepest_state - Set/clear governor override flag. -* @enable: New value of the flag. +* @latency_limit_ns: A latency limit constraint * -* Set/unset the current CPU to use the deepest idle state (override governors -* going forward if set). +* Set/unset the current CPU to use the deepest idle state with the exit +* latency within @latency_limit_ns */ -void cpuidle_use_deepest_state (bool enable) +void cpuidle_use_deepest_state (u64 latency_limit_ns) {struct cpuidle_device *dev;preempt_disable (); dev =cpuidle_get_device ();if (dev) - dev->use_deepest_state = enable; + dev->forced_idle_latency_limit_ns = latency_limit_ns;preempt_enable (); }
include/linux/cpu.h +5 -1
@@ -179,7 +179,11 @@ void arch_cpu_idle_dead(void);int cpu_report_state (int cpu);int cpu_check_up_prepare (int cpu);void cpu_set_state_online (int cpu); -void play_idle (unsigned long duration_us); +void play_idle_precise (u64 duration_ns,u64 latency_ns); +static inline void play_idle (unsigned long duration_us) +{ +play_idle_precise (duration_us *NSEC_PER_USEC ,U64_MAX ); +}#ifdef CONFIG_HOTPLUG_CPU bool cpu_wait_death (unsigned int cpu,int seconds);
include/linux/cpuidle.h +3 -3
@@ -83,7 +83,6 @@ struct cpuidle_driver_kobj;struct cpuidle_device {unsigned int registered:1 ;unsigned int enabled:1 ; -unsigned int use_deepest_state:1 ;unsigned int poll_time_limit:1 ;unsigned int cpu;ktime_t next_hrtimer;
@@ -91,6 +90,7 @@ struct cpuidle_device {int last_state_idx;int last_residency;u64 poll_limit_ns; +u64 forced_idle_latency_limit_ns;struct cpuidle_state_usage states_usage[CPUIDLE_STATE_MAX ];struct cpuidle_state_kobj *kobjs[CPUIDLE_STATE_MAX ];struct cpuidle_driver_kobj *kobj_driver;
@@ -210,7 +210,7 @@ extern int cpuidle_find_deepest_state(struct cpuidle_driver *drv, struct cpuidle_device *dev);extern int cpuidle_enter_s2idle (struct cpuidle_driver *drv,struct cpuidle_device *dev); -extern void cpuidle_use_deepest_state (bool enable); +extern void cpuidle_use_deepest_state (u64 latency_limit_ns);#else static inline int cpuidle_find_deepest_state (struct cpuidle_driver *drv,struct cpuidle_device *dev)
@@ -218,7 +218,7 @@ static inline int cpuidle_find_deepest_state(struct cpuidle_driver *drv,static inline int cpuidle_enter_s2idle (struct cpuidle_driver *drv,struct cpuidle_device *dev) {return -ENODEV ; } -static inline void cpuidle_use_deepest_state (bool enable) +static inline void cpuidle_use_deepest_state (u64 latency_limit_ns) { }#endif
kernel/sched/idle.c +7 -7
@@ -165,7 +165,7 @@ static void cpuidle_idle_call(void)* until a proper wakeup interrupt happens. */ -if (idle_should_enter_s2idle () || dev->use_deepest_state) { +if (idle_should_enter_s2idle () || dev->forced_idle_latency_limit_ns) {if (idle_should_enter_s2idle ()) {rcu_idle_enter ();
@@ -311,7 +311,7 @@ static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer)return HRTIMER_NORESTART ; } -void play_idle (unsigned long duration_us) +void play_idle_precise (u64 duration_ns,u64 latency_ns) {struct idle_timer it;
@@ -323,29 +323,29 @@ void play_idle(unsigned long duration_us)WARN_ON_ONCE (current->nr_cpus_allowed !=1 );WARN_ON_ONCE (!(current->flags &PF_KTHREAD ));WARN_ON_ONCE (!(current->flags &PF_NO_SETAFFINITY )); -WARN_ON_ONCE (!duration_us); +WARN_ON_ONCE (!duration_ns);rcu_sleep_check ();preempt_disable (); current->flags |=PF_IDLE ; -cpuidle_use_deepest_state (true); +cpuidle_use_deepest_state (latency_ns); it.done =0 ;hrtimer_init_on_stack (&it.timer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL ); it.timer.function = idle_inject_timer_fn; -hrtimer_start (&it.timer,ns_to_ktime (duration_us *NSEC_PER_USEC ), +hrtimer_start (&it.timer,ns_to_ktime (duration_ns),HRTIMER_MODE_REL_PINNED );while (!READ_ONCE (it.done))do_idle (); -cpuidle_use_deepest_state (false); +cpuidle_use_deepest_state (0 ); current->flags &= ~PF_IDLE ;preempt_fold_need_resched ();preempt_enable (); } -EXPORT_SYMBOL_GPL (play_idle); +EXPORT_SYMBOL_GPL (play_idle_precise);void cpu_startup_entry (enum cpuhp_state state) {
Signature · 1 line
2.17.1
PATCH v2 2/2 cpuidle: Use the latency to call find_deepest_idle_state
As the dev->forced_idle_latency_limit_ns is filled with the latency value when this function is called, use it as a parameter to the find_deepest_idle_state() function.
--- drivers/cpuidle/cpuidle.c | 7 +++++-- include/linux/cpuidle.h | 6 ++++-- kernel/sched/idle.c | 8 +++++++- 3 files changed, 16 insertions(+), 5 deletions(-)
drivers/cpuidle/cpuidle.c +5 -2
@@ -122,9 +122,12 @@ void cpuidle_use_deepest_state(u64 latency_limit_ns)* @dev: cpuidle device for the given CPU. */ int cpuidle_find_deepest_state (struct cpuidle_driver *drv, -struct cpuidle_device *dev) +struct cpuidle_device *dev, +u64 latency_limit_ns) { -return find_deepest_state (drv, dev,UINT_MAX ,0 , false); +return find_deepest_state (drv, dev, +div_u64 (latency_limit_ns,NSEC_PER_USEC ), +0 , false); }#ifdef CONFIG_SUSPEND
include/linux/cpuidle.h +4 -2
@@ -207,13 +207,15 @@ static inline struct cpuidle_device *cpuidle_get_device(void) {return NULL; }#ifdef CONFIG_CPU_IDLE extern int cpuidle_find_deepest_state (struct cpuidle_driver *drv, -struct cpuidle_device *dev); +struct cpuidle_device *dev, +u64 latency_limit_ns);extern int cpuidle_enter_s2idle (struct cpuidle_driver *drv,struct cpuidle_device *dev);extern void cpuidle_use_deepest_state (u64 latency_limit_ns);#else static inline int cpuidle_find_deepest_state (struct cpuidle_driver *drv, -struct cpuidle_device *dev) +struct cpuidle_device *dev, +u64 latency_limit_ns) {return -ENODEV ; }static inline int cpuidle_enter_s2idle (struct cpuidle_driver *drv,struct cpuidle_device *dev)
kernel/sched/idle.c +7 -1
@@ -166,6 +166,9 @@ static void cpuidle_idle_call(void)*/ if (idle_should_enter_s2idle () || dev->forced_idle_latency_limit_ns) { + +u64 latency_limit_ns = dev->forced_idle_latency_limit_ns; +if (idle_should_enter_s2idle ()) {rcu_idle_enter ();
@@ -176,12 +179,15 @@ static void cpuidle_idle_call(void) }rcu_idle_exit (); + + latency_limit_ns =U64_MAX ; }tick_nohz_idle_stop_tick ();rcu_idle_enter (); - next_state =cpuidle_find_deepest_state (drv, dev); + next_state =cpuidle_find_deepest_state (drv, dev, + latency_limit_ns);call_cpuidle (drv, dev, next_state); }else {bool stop_tick = true;
Signature · 1 line
2.17.1
Quoted 157 lines · CET Daniel Lezcano
We want to specify a latency constraint when choosing an idle state at
play_idletime. Instead of duplicating the information in the structure or propagate the latency in the call stack, change theuse_deepest_statebyforced_latency_limit_nsto introduce this constraint. The idea being that when it is set, idle is forced (i.e. no governors), but there is a latency limit for the state to use.A zero latency constraint means "do not use the deepest idle state path" as the '
use_deepest_state' boolean was used in thecpuidle_idle_call.In addition add the
play_idle_precise()function getting a nanosec base latency constraint as parameter. Theplay_idle()function becomes a wrapper passing anU64_MAXtoplay_idle_precise().Suggested-by Rafael J. Wysocki <[email protected]>Signed-off-by Daniel Lezcano <[email protected]>--- drivers/cpuidle/cpuidle.c | 10 +++++----- include/linux/cpu.h | 6 +++++- include/linux/cpuidle.h | 6 +++--- kernel/sched/idle.c | 14 +++++++------- 4 files changed, 20 insertions(+), 16 deletions(-)drivers/cpuidle/cpuidle.c +5
-5@@ -100,19 +100,19 @@ static int find_deepest_state(struct cpuidle_driver *drv,/** * cpuidle_use_deepest_state - Set/clear governor override flag. -* @enable: New value of the flag. +* @latency_limit_ns: A latency limit constraint * -* Set/unset the current CPU to use the deepest idle state (override governors -* going forward if set). +* Set/unset the current CPU to use the deepest idle state with the exit +* latency within @latency_limit_ns */ -void cpuidle_use_deepest_state (bool enable) +void cpuidle_use_deepest_state (u64 latency_limit_ns) {struct cpuidle_device *dev;preempt_disable (); dev =cpuidle_get_device ();if (dev) - dev->use_deepest_state = enable; + dev->forced_idle_latency_limit_ns = latency_limit_ns;preempt_enable (); }include/linux/cpu.h +5
-1@@ -179,7 +179,11 @@ void arch_cpu_idle_dead(void);int cpu_report_state (int cpu);int cpu_check_up_prepare (int cpu);void cpu_set_state_online (int cpu); -void play_idle (unsigned long duration_us); +void play_idle_precise (u64 duration_ns,u64 latency_ns); +static inline void play_idle (unsigned long duration_us) +{ +play_idle_precise (duration_us *NSEC_PER_USEC ,U64_MAX ); +}#ifdef CONFIG_HOTPLUG_CPU bool cpu_wait_death (unsigned int cpu,int seconds);include/linux/cpuidle.h +3
-3@@ -83,7 +83,6 @@ struct cpuidle_driver_kobj;struct cpuidle_device {unsigned int registered:1 ;unsigned int enabled:1 ; -unsigned int use_deepest_state:1 ;unsigned int poll_time_limit:1 ;unsigned int cpu;ktime_t next_hrtimer;@@ -91,6 +90,7 @@ struct cpuidle_device {int last_state_idx;int last_residency;u64 poll_limit_ns; +u64 forced_idle_latency_limit_ns;struct cpuidle_state_usage states_usage[CPUIDLE_STATE_MAX ];struct cpuidle_state_kobj *kobjs[CPUIDLE_STATE_MAX ];struct cpuidle_driver_kobj *kobj_driver;@@ -210,7 +210,7 @@ extern int cpuidle_find_deepest_state(struct cpuidle_driver *drv, struct cpuidle_device *dev);extern int cpuidle_enter_s2idle (struct cpuidle_driver *drv,struct cpuidle_device *dev); -extern void cpuidle_use_deepest_state (bool enable); +extern void cpuidle_use_deepest_state (u64 latency_limit_ns);#else static inline int cpuidle_find_deepest_state (struct cpuidle_driver *drv,struct cpuidle_device *dev)@@ -218,7 +218,7 @@ static inline int cpuidle_find_deepest_state(struct cpuidle_driver *drv,static inline int cpuidle_enter_s2idle (struct cpuidle_driver *drv,struct cpuidle_device *dev) {return -ENODEV ; } -static inline void cpuidle_use_deepest_state (bool enable) +static inline void cpuidle_use_deepest_state (u64 latency_limit_ns) { }#endif kernel/sched/idle.c +6
-6@@ -165,7 +165,7 @@ static void cpuidle_idle_call(void)* until a proper wakeup interrupt happens. */ -if (idle_should_enter_s2idle () || dev->use_deepest_state) { +if (idle_should_enter_s2idle () || dev->forced_idle_latency_limit_ns) {if (idle_should_enter_s2idle ()) {rcu_idle_enter ();@@ -311,7 +311,7 @@ static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer)return HRTIMER_NORESTART ; } -void play_idle (unsigned long duration_us) +void play_idle_precise (u64 duration_ns,u64 latency_ns) {struct idle_timer it;@@ -323,29 +323,29 @@ void play_idle(unsigned long duration_us)WARN_ON_ONCE (current->nr_cpus_allowed !=1 );WARN_ON_ONCE (!(current->flags &PF_KTHREAD ));WARN_ON_ONCE (!(current->flags &PF_NO_SETAFFINITY )); -WARN_ON_ONCE (!duration_us); +WARN_ON_ONCE (!duration_ns);rcu_sleep_check ();preempt_disable (); current->flags |=PF_IDLE ; -cpuidle_use_deepest_state (true); +cpuidle_use_deepest_state (latency_ns); it.done =0 ;hrtimer_init_on_stack (&it.timer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL ); it.timer.function = idle_inject_timer_fn; -hrtimer_start (&it.timer,ns_to_ktime (duration_us *NSEC_PER_USEC ), +hrtimer_start (&it.timer,ns_to_ktime (duration_ns),HRTIMER_MODE_REL_PINNED );while (!READ_ONCE (it.done))do_idle (); -cpuidle_use_deepest_state (false); +cpuidle_use_deepest_state (0 );
kernel/sched/idle.c void play_idle(unsigned long duration_us) · line 351
Quoted 10 lines
kernel/sched/idle.c +1
-1current->flags &= ~PF_IDLE ;preempt_fold_need_resched ();preempt_enable (); } -EXPORT_SYMBOL_GPL (play_idle); +EXPORT_SYMBOL_GPL (play_idle_precise);void cpu_startup_entry (enum cpuhp_state state) {
Queued up for 5.5 along with the [2/2], but I have made quite a few changes to the patches, so please double check the result in my bleeding-edge branch.
Thanks!
Quoted 20 lines · Rafael J. Wysocki
Quoted 19 lines · CET Daniel Lezcano
We want to specify a latency constraint when choosing an idle state at
play_idletime. Instead of duplicating the information in the structure or propagate the latency in the call stack, change theuse_deepest_statebyforced_latency_limit_nsto introduce this constraint. The idea being that when it is set, idle is forced (i.e. no governors), but there is a latency limit for the state to use.A zero latency constraint means "do not use the deepest idle state path" as the '
use_deepest_state' boolean was used in thecpuidle_idle_call.In addition add the
play_idle_precise()function getting a nanosec base latency constraint as parameter. Theplay_idle()function becomes a wrapper passing anU64_MAXtoplay_idle_precise().Suggested-by Rafael J. Wysocki <[email protected]>Signed-off-by Daniel Lezcano <[email protected]>---
[ ... ]
Quoted 4 lines
Queued up for 5.5 along with the [2/2], but I have made quite a few changes to the patches, so please double check the result in my bleeding-edge branch.
Thanks!
They look good to me.
Thanks
-- Daniel
Signature · 5 lines
<http://www.linaro.org/> Linaro.org │ Open source software for ARM SoCs Follow Linaro: <http://www.facebook.com/pages/Linaro> Facebook | <http://twitter.com/#!/linaroorg> Twitter | <http://www.linaro.org/linaro-blog/> Blog