cpuidle: Handle TIF_NR_POLLING on behalf of polling idle states
Subsystems (10)
v3 cover cpuidle: Handle TIF_NR_POLLING on behalf of polling idle states
The TIF_NR_POLLING handling against TIF_NEED_RESCHED polling/monitoring idle states (mwait and also software polling) is a bit messy, with quite some wasted cycles spent on useless atomic operations. This tries to consolidate this state handling from the cpuidle core.
Changes since v2:
_ Handle buggy mwait implementations (thanks Rafael)
Frederic Weisbecker (4):
cpuidle: Remove unnecessary current_clr_polling_and_test() from
haltpoll
x86/cpuidle: Move buggy mwait implementations away from
CPUIDLE_FLAG_MWAIT
cpuidle: Remove call_cpuidle_s2idle()
cpuidle: Handle TIF_NR_POLLING on behalf of software polling idle
states
Peter Zijlstra (2):
cpuidle: Introduce CPUIDLE_FLAG_MWAIT
cpuidle: Handle TIF_NR_POLLING on behalf of CPUIDLE_FLAG_MWAIT states
arch/arm/include/asm/cpuidle.h | 2 ++ arch/arm64/include/asm/cpuidle.h | 3 ++ arch/powerpc/include/asm/cpuidle.h | 4 +++ arch/riscv/include/asm/cpuidle.h | 2 ++ arch/x86/include/asm/cpuidle.h | 12 +++++++ arch/x86/include/asm/mwait.h | 27 +++++++-------- drivers/acpi/processor_idle.c | 5 +++ drivers/cpuidle/cpuidle-haltpoll.c | 3 -- drivers/cpuidle/cpuidle-powernv.c | 10 ------ drivers/cpuidle/cpuidle-pseries.c | 11 ------- drivers/cpuidle/cpuidle.c | 22 ++++++++++++- drivers/cpuidle/poll_state.c | 30 +++++++---------- drivers/idle/intel_idle.c | 8 +++++ include/asm-generic/Kbuild | 1 + include/asm-generic/cpuidle.h | 10 ++++++ include/linux/cpuidle.h | 1 + include/linux/sched/idle.h | 7 +++- kernel/sched/idle.c | 53 +++++++++--------------------- 18 files changed, 114 insertions(+), 97 deletions(-) create mode 100644 arch/x86/include/asm/cpuidle.h create mode 100644 include/asm-generic/cpuidle.h
Signature · 1 line
2.46.0
PATCH v3 1/6 cpuidle: Remove unnecessary current_clr_polling_and_test() from haltpoll
When cpuidle drivers ->enter() callback are called, the TIF_NR_POLLING flag is cleared already and TIF_NEED_RESCHED checked by call_cpuidle().
Therefore calling current_clr_polling_and_test() is redundant here and further setting of TIF_NEED_RESCHED will result in an IPI and thus an idle loop exit. This call can be safely removed.
--- drivers/cpuidle/cpuidle-haltpoll.c | 3 --- 1 file changed, 3 deletions(-)
drivers/cpuidle/cpuidle-haltpoll.c +0 -3
@@ -28,9 +28,6 @@ static enum cpuhp_state haltpoll_hp_state; static __cpuidle int default_enter_idle(struct cpuidle_device *dev,struct cpuidle_driver *drv,int index) { -if (current_clr_polling_and_test ()) -return index; -arch_cpu_idle ();return index; }
Signature · 1 line
2.46.0
PATCH v3 2/6 cpuidle: Introduce CPUIDLE_FLAG_MWAIT
In reply to Frederic WeisbeckerFrom: Peter Zijlstra <[email protected]>
Provide a way to tell the cpuidle core about states monitoring TIF_NEED_RESCHED on the hardware level, monitor/mwait users being the only examples in use.
This will allow cpuidle core to manage TIF_NR_POLLING on behalf of all kinds of TIF_NEED_RESCHED watching states while keeping a necessary distinction for the governors between software loops polling on TIF_NEED_RESCHED and hardware monitored writes to thread flags.
[fweisbec: _ Initialize flag from acpi_processor_setup_cstates() instead
of acpi_processor_setup_lpi_states(), as the latter seem to be about arm64...
_ Rename CPUIDLE_FLAG_NO_IPI to CPUIDLE_FLAG_MWAIT]
--- drivers/acpi/processor_idle.c | 3 +++ drivers/idle/intel_idle.c | 5 ++++- include/linux/cpuidle.h | 1 + 3 files changed, 8 insertions(+), 1 deletion(-)
drivers/acpi/processor_idle.c +3 -0
@@ -806,6 +806,9 @@ static int acpi_processor_setup_cstates(struct acpi_processor *pr)if (cx->type ==ACPI_STATE_C1 || cx->type ==ACPI_STATE_C2 ) drv->safe_state_index = count; +if (cx->entry_method ==ACPI_CSTATE_FFH ) + state->flags |=CPUIDLE_FLAG_MWAIT ; +/* * Halt-induced C1 is not good for ->enter_s2idle, because it * re-enables interrupts on exit. Moreover, C1 is generally not
drivers/idle/intel_idle.c +4 -1
@@ -1787,7 +1787,8 @@ static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv)if (cx->type >ACPI_STATE_C1 ) state->target_residency *=3 ; - state->flags =MWAIT2flg (cx->address); + state->flags =MWAIT2flg (cx->address) |CPUIDLE_FLAG_MWAIT ; +if (cx->type >ACPI_STATE_C2 ) state->flags |=CPUIDLE_FLAG_TLB_FLUSHED ;
@@ -2072,6 +2073,8 @@ static bool __init intel_idle_verify_cstate(unsigned int mwait_hint)static void state_update_enter_method (struct cpuidle_state *state,int cstate) { + state->flags |=CPUIDLE_FLAG_MWAIT ; +if (state->flags &CPUIDLE_FLAG_INIT_XSTATE ) {/* * Combining with XSTATE with IBRS or IRQ_ENABLE flags
include/linux/cpuidle.h +1 -0
@@ -85,6 +85,7 @@ struct cpuidle_state {#define CPUIDLE_FLAG_OFF BIT(4)/* disable this state by default */ #define CPUIDLE_FLAG_TLB_FLUSHED BIT(5)/* idle-state flushes TLBs */ #define CPUIDLE_FLAG_RCU_IDLE BIT(6)/* idle-state takes care of RCU */ +#define CPUIDLE_FLAG_MWAIT BIT(7)/* hardware need_resched() monitoring */ struct cpuidle_device_kobj ;struct cpuidle_state_kobj ;
Signature · 1 line
2.46.0
PATCH v3 3/6 x86/cpuidle: Move buggy mwait implementations away from CPUIDLE_FLAG_MWAIT
In reply to Frederic WeisbeckerBuggy MWAIT implementations can't carry the CPUIDLE_FLAG_MWAIT flag because they require IPIs to wake up. Therefore they shouldn't be called with TIF_NR_POLLING.
--- arch/arm/include/asm/cpuidle.h | 2 ++ arch/arm64/include/asm/cpuidle.h | 3 +++ arch/powerpc/include/asm/cpuidle.h | 4 ++++ arch/riscv/include/asm/cpuidle.h | 2 ++ arch/x86/include/asm/cpuidle.h | 12 ++++++++++++ drivers/acpi/processor_idle.c | 4 +++- drivers/idle/intel_idle.c | 9 +++++++-- include/asm-generic/Kbuild | 1 + include/asm-generic/cpuidle.h | 10 ++++++++++ 9 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 arch/x86/include/asm/cpuidle.h create mode 100644 include/asm-generic/cpuidle.h
arch/arm/include/asm/cpuidle.h +2 -0
@@ -55,4 +55,6 @@ struct arm_cpuidle_irq_context { };#define arm_cpuidle_save_irq_context (c) (void)c#define arm_cpuidle_restore_irq_context (c) (void)c +#include <asm-generic/cpuidle.h> +#endif
arch/arm64/include/asm/cpuidle.h +3 -0
@@ -38,4 +38,7 @@ struct arm_cpuidle_irq_context { };#define arm_cpuidle_save_irq_context (c) (void)c#define arm_cpuidle_restore_irq_context (c) (void)c#endif + +#include <asm-generic/cpuidle.h> +#endif
arch/powerpc/include/asm/cpuidle.h +4 -0
@@ -102,4 +102,8 @@ static inline void report_invalid_psscr_val(u64 psscr_val, int err)#endif +#ifndef __ASSEMBLY__ +#include <asm-generic/cpuidle.h> +#endif +#endif
arch/riscv/include/asm/cpuidle.h +2 -0
@@ -21,4 +21,6 @@ static inline void cpu_do_idle(void) wait_for_interrupt(); } +#include <asm-generic/cpuidle.h> +#endif
arch/x86/include/asm/cpuidle.h new file +12 -0
@@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_CPUIDLE_H +#define _ASM_X86_CPUIDLE_H + +#include <asm/cpufeature.h> + +static inline bool arch_cpuidle_mwait_needs_ipi (void ) +{ +return boot_cpu_has_bug (X86_BUG_MONITOR ); +} + +#endif /* _ASM_X86_CPUIDLE_H */
drivers/acpi/processor_idle.c +3 -1
@@ -23,6 +23,7 @@#include <linux/perf_event.h> #include <acpi/processor.h> #include <linux/context_tracking.h> +#include <asm/cpuidle.h> /* * Include the apic definitions for x86 to have the APIC timer related defines
@@ -806,7 +807,8 @@ static int acpi_processor_setup_cstates(struct acpi_processor *pr)if (cx->type ==ACPI_STATE_C1 || cx->type ==ACPI_STATE_C2 ) drv->safe_state_index = count; -if (cx->entry_method ==ACPI_CSTATE_FFH ) +if (cx->entry_method ==ACPI_CSTATE_FFH && + !arch_cpuidle_mwait_needs_ipi ()) state->flags |=CPUIDLE_FLAG_MWAIT ;/*
drivers/idle/intel_idle.c +7 -2
@@ -56,6 +56,7 @@#include <asm/mwait.h> #include <asm/spec-ctrl.h> #include <asm/fpu/api.h> +#include <asm/cpuidle.h> #define INTEL_IDLE_VERSION "0.5.1"
@@ -1787,7 +1788,10 @@ static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv)if (cx->type >ACPI_STATE_C1 ) state->target_residency *=3 ; - state->flags =MWAIT2flg (cx->address) |CPUIDLE_FLAG_MWAIT ; + state->flags =MWAIT2flg (cx->address); + +if (!arch_cpuidle_mwait_needs_ipi ()) + state->flags |=CPUIDLE_FLAG_MWAIT ;if (cx->type >ACPI_STATE_C2 ) state->flags |=CPUIDLE_FLAG_TLB_FLUSHED ;
@@ -2073,7 +2077,8 @@ static bool __init intel_idle_verify_cstate(unsigned int mwait_hint)static void state_update_enter_method (struct cpuidle_state *state,int cstate) { - state->flags |=CPUIDLE_FLAG_MWAIT ; +if (!arch_cpuidle_mwait_needs_ipi ()) + state->flags |=CPUIDLE_FLAG_MWAIT ;if (state->flags &CPUIDLE_FLAG_INIT_XSTATE ) {/*
include/asm-generic/Kbuild +1 -0
@@ -13,6 +13,7 @@ mandatory-y += cacheflush.h mandatory-y += cfi.h mandatory-y += checksum.h mandatory-y += compat.h +mandatory-y += cpuidle.h mandatory-y += current.h mandatory-y += delay.h mandatory-y += device.h
include/asm-generic/cpuidle.h new file +10 -0
@@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_CPUIDLE_H +#define __ASM_CPUIDLE_H + +static inline bool arch_cpuidle_mwait_needs_ipi (void ) +{ +return true; +} + +#endif /* __ASM_CPUIDLE_H */
Signature · 1 line
2.46.0
PATCH v3 4/6 cpuidle: Handle TIF_NR_POLLING on behalf of CPUIDLE_FLAG_MWAIT states
In reply to Frederic WeisbeckerFrom: Peter Zijlstra <[email protected]>
The current handling of TIF_NR_POLLING is a bit of a maze:
1) TIF_NR_POLLING is set on idle entry (one atomic set)
2) Once cpuidle has selected an appropriate state and the tick is evaluated and then possibly stopped, TIF_NR_POLLING is cleared (one RmW operation)
2) The cpuidle state is then called with TIF_NR_POLLING cleared but if the state polls on (or monitors) need_resched() it sets again TIF_NR_POLLING before sleeping and clears it on wake-up. Summary:
another pair of set/clear
3) Set back TIF_NR_POLLING (one atomic set)
4) goto 2) if need_resched() is not set
All those costly atomic operations, fully ordered RmW for some of them, could be avoided if the cpuidle core knew in advance if the target state polls on (or monitors) need_resched(). If so, TIF_NR_POLLING could simply be set once upon entering the idle loop and cleared once after idle loop exit.
Start dealing with that with handling TIF_NR_POLLING on behalf of mwait based states.
[fweisbec: _ Handle broadcast properly
_ Ignore mwait_idle() as it can be used by default_idle_call()]
--- arch/x86/include/asm/mwait.h | 27 +++++++++++------------- drivers/cpuidle/cpuidle.c | 22 +++++++++++++++++++- include/linux/sched/idle.h | 7 ++++++- kernel/sched/idle.c | 40 +++++++++++++----------------------- 4 files changed, 53 insertions(+), 43 deletions(-)
arch/x86/include/asm/mwait.h +12 -15
@@ -116,25 +116,22 @@ static __always_inline void __sti_mwait(unsigned long eax, unsigned long ecx)*/ static __always_inline void mwait_idle_with_hints(unsigned long eax, unsigned long ecx) { -if (static_cpu_has_bug (X86_BUG_MONITOR ) || !current_set_polling_and_test ()) { -if (static_cpu_has_bug (X86_BUG_CLFLUSH_MONITOR )) { -mb (); -clflush ((void *)¤t_thread_info ()->flags); -mb (); - } +if (static_cpu_has_bug (X86_BUG_CLFLUSH_MONITOR )) { +mb (); +clflush ((void *)¤t_thread_info ()->flags); +mb (); + } -__monitor ((void *)¤t_thread_info ()->flags,0 ,0 ); +__monitor ((void *)¤t_thread_info ()->flags,0 ,0 ); -if (!need_resched ()) { -if (ecx &1 ) { -__mwait (eax, ecx); - }else { -__sti_mwait (eax, ecx); -raw_local_irq_disable (); - } +if (!need_resched ()) { +if (ecx &1 ) { +__mwait (eax, ecx); + }else { +__sti_mwait (eax, ecx); +raw_local_irq_disable (); } } -current_clr_polling (); }/*
drivers/cpuidle/cpuidle.c +21 -1
@@ -217,10 +217,10 @@ noinstr int cpuidle_enter_state(struct cpuidle_device *dev, int index) {int entered_state; -struct cpuidle_state *target_state = &drv->states[index];bool broadcast = !!(target_state->flags &CPUIDLE_FLAG_TIMER_STOP );ktime_t time_start, time_end; +bool polling;instrumentation_begin ();
@@ -237,6 +237,23 @@ noinstr int cpuidle_enter_state(struct cpuidle_device *dev, broadcast = false; } + polling = target_state->flags &CPUIDLE_FLAG_MWAIT ; + +/* +* If the target state doesn't poll on need_resched(), this is +* the last check after which further TIF_NEED_RESCHED remote setting +* will involve an IPI. +*/ +if (!polling &¤t_clr_polling_and_test ()) { +if (broadcast) +tick_broadcast_exit (); + dev->last_residency_ns =0 ; +local_irq_enable (); +instrumentation_end (); +return -EBUSY ; + } + +if (target_state->flags &CPUIDLE_FLAG_TLB_FLUSHED )leave_mm ();
@@ -336,6 +353,9 @@ noinstr int cpuidle_enter_state(struct cpuidle_device *dev, dev->states_usage[index].rejected++; } +if (!polling) +__current_set_polling (); +instrumentation_end ();return entered_state;
include/linux/sched/idle.h +6 -1
@@ -68,6 +68,8 @@ static __always_inline bool __must_check current_set_polling_and_test(void) static __always_inline bool __must_check current_clr_polling_and_test(void) { +bool ret; +__current_clr_polling ();/*
@@ -76,7 +78,10 @@ static __always_inline bool __must_check current_clr_polling_and_test(void)*/ smp_mb__after_atomic (); - return unlikely(tif_need_resched()); + ret =unlikely (tif_need_resched ()); +if (ret) +__current_set_polling (); + return ret; }#else
kernel/sched/idle.c +14 -26
@@ -114,12 +114,13 @@ void __cpuidle default_idle_call(void)stop_critical_timings ();ct_cpuidle_enter (); -arch_cpu_idle (); +arch_cpu_idle ();// XXX assumes !polling ct_cpuidle_exit ();start_critical_timings ();trace_cpu_idle (PWR_EVENT_EXIT ,smp_processor_id ());cond_tick_broadcast_exit (); +__current_set_polling (); }local_irq_enable ();instrumentation_end ();
@@ -128,31 +129,14 @@ void __cpuidle default_idle_call(void)static int call_cpuidle_s2idle (struct cpuidle_driver *drv,struct cpuidle_device *dev) { +int ret; +if (current_clr_polling_and_test ())return -EBUSY ; -return cpuidle_enter_s2idle (drv, dev); -} - -static int call_cpuidle (struct cpuidle_driver *drv,struct cpuidle_device *dev, -int next_state) -{ -/* -* The idle task must be scheduled, it is pointless to go to idle, just -* update no idle residency and return. -*/ -if (current_clr_polling_and_test ()) { - dev->last_residency_ns =0 ; -local_irq_enable (); -return -EBUSY ; - } - -/* -* Enter the idle state previously returned by the governor decision. -* This function will block until an interrupt occurs and will take -* care of re-enabling the local interrupts -*/ -return cpuidle_enter (drv, dev, next_state); + ret =cpuidle_enter_s2idle (drv, dev); +__current_set_polling (); +return ret; }/**
@@ -213,7 +197,7 @@ static void cpuidle_idle_call(void)tick_nohz_idle_stop_tick (); next_state =cpuidle_find_deepest_state (drv, dev, max_latency_ns); -call_cpuidle (drv, dev, next_state); +cpuidle_enter (drv, dev, next_state); }else {bool stop_tick = true;
@@ -227,7 +211,12 @@ static void cpuidle_idle_call(void)else tick_nohz_idle_retain_tick (); - entered_state =call_cpuidle (drv, dev, next_state); +/* +* Enter the idle state previously returned by the governor decision. +* This function will block until an interrupt occurs and will take +* care of re-enabling the local interrupts. +*/ + entered_state =cpuidle_enter (drv, dev, next_state);/* * Give the governor an opportunity to reflect on the outcome */
@@ -235,7 +224,6 @@ static void cpuidle_idle_call(void) } exit_idle: -__current_set_polling ();/* * It is up to the idle functions to re-enable local interrupts
Signature · 1 line
2.46.0
PATCH v3 5/6 cpuidle: Remove call_cpuidle_s2idle()
In reply to Frederic WeisbeckerThis middle call is unecessary, especially now that its counterpart call_cpuidle() has been removed.
--- kernel/sched/idle.c | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-)
kernel/sched/idle.c +6 -17
@@ -126,19 +126,6 @@ void __cpuidle default_idle_call(void)instrumentation_end (); } -static int call_cpuidle_s2idle (struct cpuidle_driver *drv, -struct cpuidle_device *dev) -{ -int ret; - -if (current_clr_polling_and_test ()) -return -EBUSY ; - - ret =cpuidle_enter_s2idle (drv, dev); -__current_set_polling (); -return ret; -} -/** * cpuidle_idle_call - the main idle function *
@@ -184,10 +171,12 @@ static void cpuidle_idle_call(void)u64 max_latency_ns;if (idle_should_enter_s2idle ()) { - - entered_state =call_cpuidle_s2idle (drv, dev); -if (entered_state >0 ) - goto exit_idle; +if (!current_clr_polling_and_test ()) { + entered_state =cpuidle_enter_s2idle (drv, dev); +__current_set_polling (); +if (entered_state >0 ) + goto exit_idle; + } max_latency_ns =U64_MAX ; }else {
Signature · 1 line
2.46.0
PATCH v3 6/6 cpuidle: Handle TIF_NR_POLLING on behalf of software polling idle states
In reply to Frederic WeisbeckerSoftware polling idle states set again TIF_NR_POLLING and clear it upon exit. This involves error prone duplicated code and wasted cycles performing atomic operations, sometimes RmW fully ordered.
To avoid this, benefit instead from the same generic TIF_NR_POLLING handling that is currently in use for hardware monitoring states.
--- drivers/cpuidle/cpuidle-powernv.c | 10 ---------- drivers/cpuidle/cpuidle-pseries.c | 11 ----------- drivers/cpuidle/cpuidle.c | 2 +- drivers/cpuidle/poll_state.c | 30 ++++++++++++------------------ 4 files changed, 13 insertions(+), 40 deletions(-)
drivers/cpuidle/cpuidle-powernv.c +0 -10
@@ -71,8 +71,6 @@ static int snooze_loop(struct cpuidle_device *dev, {u64 snooze_exit_time; -set_thread_flag (TIF_POLLING_NRFLAG ); -local_irq_enable (); snooze_exit_time =get_tb () +get_snooze_timeout (dev, drv, index);
@@ -81,21 +79,13 @@ static int snooze_loop(struct cpuidle_device *dev,HMT_very_low ();while (!need_resched ()) {if (likely (snooze_timeout_en) &&get_tb () > snooze_exit_time) { -/* -* Task has not woken up but we are exiting the polling -* loop anyway. Require a barrier after polling is -* cleared to order subsequent test of need_resched(). -*/ -clear_thread_flag (TIF_POLLING_NRFLAG ); dev->poll_time_limit = true; -smp_mb ();break ; } }HMT_medium ();ppc64_runlatch_on (); -clear_thread_flag (TIF_POLLING_NRFLAG );local_irq_disable ();
drivers/cpuidle/cpuidle-pseries.c +0 -11
@@ -40,8 +40,6 @@ int snooze_loop(struct cpuidle_device *dev, struct cpuidle_driver *drv, {u64 snooze_exit_time; -set_thread_flag (TIF_POLLING_NRFLAG ); -pseries_idle_prolog ();raw_local_irq_enable (); snooze_exit_time =get_tb () + snooze_timeout;
@@ -51,21 +49,12 @@ int snooze_loop(struct cpuidle_device *dev, struct cpuidle_driver *drv,HMT_low ();HMT_very_low ();if (likely (snooze_timeout_en) &&get_tb () > snooze_exit_time) { -/* -* Task has not woken up but we are exiting the polling -* loop anyway. Require a barrier after polling is -* cleared to order subsequent test of need_resched(). -*/ dev->poll_time_limit = true; -clear_thread_flag (TIF_POLLING_NRFLAG ); -smp_mb ();break ; } }HMT_medium (); -clear_thread_flag (TIF_POLLING_NRFLAG ); -raw_local_irq_disable ();pseries_idle_epilog ();
drivers/cpuidle/cpuidle.c +1 -1
@@ -237,7 +237,7 @@ noinstr int cpuidle_enter_state(struct cpuidle_device *dev, broadcast = false; } - polling = target_state->flags &CPUIDLE_FLAG_MWAIT ; + polling = target_state->flags & (CPUIDLE_FLAG_MWAIT |CPUIDLE_FLAG_POLLING );/* * If the target state doesn't poll on need_resched(), this is
drivers/cpuidle/poll_state.c +12 -18
@@ -13,35 +13,29 @@ static int __cpuidle poll_idle(struct cpuidle_device *dev,struct cpuidle_driver *drv,int index) { -u64 time_start; - - time_start =local_clock_noinstr (); +u64 time_start =local_clock_noinstr (); +unsigned int loop_count =0 ; +u64 limit; dev->poll_time_limit = false;raw_local_irq_enable (); -if (!current_set_polling_and_test ()) { -unsigned int loop_count =0 ; -u64 limit; - limit =cpuidle_poll_time (drv, dev); + limit =cpuidle_poll_time (drv, dev); -while (!need_resched ()) { -cpu_relax (); -if (loop_count++ <POLL_IDLE_RELAX_COUNT ) -continue ; +while (!need_resched ()) { +cpu_relax (); +if (loop_count++ <POLL_IDLE_RELAX_COUNT ) +continue ; - loop_count =0 ; -if (local_clock_noinstr () - time_start > limit) { - dev->poll_time_limit = true; -break ; - } + loop_count =0 ; +if (local_clock_noinstr () - time_start > limit) { + dev->poll_time_limit = true; +break ; } }raw_local_irq_disable (); -current_clr_polling (); -return index; }
Signature · 1 line
2.46.0
Rafael J. Wysocki
maintainerRe: [PATCH 2/6] cpuidle: Introduce CPUIDLE_FLAG_MWAIT
In reply to Frederic WeisbeckerQuoted 30 lines · Frederic Weisbecker
From: Peter Zijlstra <[email protected]>
Provide a way to tell the cpuidle core about states monitoring
TIF_NEED_RESCHEDon the hardware level, monitor/mwait users being the only examples in use.This will allow cpuidle core to manage
TIF_NR_POLLINGon behalf of all kinds ofTIF_NEED_RESCHEDwatching states while keeping a necessary distinction for the governors between software loops polling onTIF_NEED_RESCHEDand hardware monitored writes to thread flags.[fweisbec: _ Initialize flag from
acpi_processor_setup_cstates()instead
ofacpi_processor_setup_lpi_states(), as the latter seem to be about arm64...
_ RenameCPUIDLE_FLAG_NO_IPItoCPUIDLE_FLAG_MWAIT]Signed-off-by Peter Zijlstra <[email protected]>Signed-off-by Frederic Weisbecker <[email protected]>--- drivers/acpi/processor_idle.c | 3 +++ drivers/idle/intel_idle.c | 5 ++++- include/linux/cpuidle.h | 1 + 3 files changed, 8 insertions(+), 1 deletion(-)drivers/acpi/processor_idle.c +0
-0
drivers/acpi/processor_idle.c static int acpi_processor_setup_cstates(struct acpi_processor *pr) · line 810
Quoted 5 lines
drivers/acpi/processor_idle.c +2
-0@@ -806,6 +806,9 @@ static int acpi_processor_setup_cstates(struct acpi_processor *pr)if (cx->type ==ACPI_STATE_C1 || cx->type ==ACPI_STATE_C2 ) drv->safe_state_index = count; +if (cx->entry_method ==ACPI_CSTATE_FFH ) + state->flags |=CPUIDLE_FLAG_MWAIT ;
FFH need not mean MWAIT in principle.
FFH in _CST means MWAIT in practice because _CST is used on x86 which implements FFH through MWAIT, but it would be good at least to put a comment here to explain that this code is only expected to run on x86.
Or better still, add something like acpi_arch_idle_state_flags(u8 entry_method) that will return CPUIDLE_FLAG_MWAIT for ACPI_CSTATE_FFH and 0 otherwise and then do
state->flags |= acpi_arch_idle_state_flags(cx->entry_method);
Quoted 40 lines
drivers/acpi/processor_idle.c +1
-0+/* * Halt-induced C1 is not good for ->enter_s2idle, because it * re-enables interrupts on exit. Moreover, C1 is generally not drivers/idle/intel_idle.c +4
-1@@ -1787,7 +1787,8 @@ static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv)if (cx->type >ACPI_STATE_C1 ) state->target_residency *=3 ; - state->flags =MWAIT2flg (cx->address); + state->flags =MWAIT2flg (cx->address) |CPUIDLE_FLAG_MWAIT ; +if (cx->type >ACPI_STATE_C2 ) state->flags |=CPUIDLE_FLAG_TLB_FLUSHED ;@@ -2072,6 +2073,8 @@ static bool __init intel_idle_verify_cstate(unsigned int mwait_hint)static void state_update_enter_method (struct cpuidle_state *state,int cstate) { + state->flags |=CPUIDLE_FLAG_MWAIT ; +if (state->flags &CPUIDLE_FLAG_INIT_XSTATE ) {/* * Combining with XSTATE with IBRS or IRQ_ENABLE flags include/linux/cpuidle.h +1
-1@@ -85,6 +85,7 @@ struct cpuidle_state {#define CPUIDLE_FLAG_OFF BIT(4)/* disable this state by default */ #define CPUIDLE_FLAG_TLB_FLUSHED BIT(5)/* idle-state flushes TLBs */ #define CPUIDLE_FLAG_RCU_IDLE BIT(6)/* idle-state takes care of RCU */ +#define CPUIDLE_FLAG_MWAIT BIT(7)/* hardware need_resched() monitoring */ struct cpuidle_device_kobj ;struct cpuidle_state_kobj ; --
Re: [PATCH 2/6] cpuidle: Introduce CPUIDLE_FLAG_MWAIT
Quoted 43 lines · Rafael J. Wysocki
Quoted 30 lines · Frederic Weisbecker
From: Peter Zijlstra <[email protected]>
Provide a way to tell the cpuidle core about states monitoring
TIF_NEED_RESCHEDon the hardware level, monitor/mwait users being the only examples in use.This will allow cpuidle core to manage
TIF_NR_POLLINGon behalf of all kinds ofTIF_NEED_RESCHEDwatching states while keeping a necessary distinction for the governors between software loops polling onTIF_NEED_RESCHEDand hardware monitored writes to thread flags.[fweisbec: _ Initialize flag from
acpi_processor_setup_cstates()instead
ofacpi_processor_setup_lpi_states(), as the latter seem to be about arm64...
_ RenameCPUIDLE_FLAG_NO_IPItoCPUIDLE_FLAG_MWAIT]Signed-off-by Peter Zijlstra <[email protected]>Signed-off-by Frederic Weisbecker <[email protected]>--- drivers/acpi/processor_idle.c | 3 +++ drivers/idle/intel_idle.c | 5 ++++- include/linux/cpuidle.h | 1 + 3 files changed, 8 insertions(+), 1 deletion(-)drivers/acpi/processor_idle.c +0
-0drivers/acpi/processor_idle.c static int acpi_processor_setup_cstates(struct acpi_processor *pr) · line 810
Quoted 5 lines
drivers/acpi/processor_idle.c +2
-0@@ -806,6 +806,9 @@ static int acpi_processor_setup_cstates(struct acpi_processor *pr)if (cx->type ==ACPI_STATE_C1 || cx->type ==ACPI_STATE_C2 ) drv->safe_state_index = count; +if (cx->entry_method ==ACPI_CSTATE_FFH ) + state->flags |=CPUIDLE_FLAG_MWAIT ;FFH need not mean MWAIT in principle.
FFH in
_CSTmeans MWAIT in practice because_CSTis used on x86 which implements FFH through MWAIT, but it would be good at least to put a comment here to explain that this code is only expected to run on x86.
Quoted 6 lines
Or better still, add something like
acpi_arch_idle_state_flags(u8 entry_method)that will returnCPUIDLE_FLAG_MWAITforACPI_CSTATE_FFHand 0 otherwise and then dostate->flags |= acpi_arch_idle_state_flags(cx->entry_method);
+1, was about to suggest the same. Though I am not aware of any Arm platforms using C-States(LPI was added to suit Arm requirements), it is better to keep the FFH definition arch specific.
Signature · 2 lines
Regards, Sudeep
Rafael J. Wysocki
maintainerRe: [PATCH 3/6] x86/cpuidle: Move buggy mwait implementations away from CPUIDLE_FLAG_MWAIT
In reply to Frederic WeisbeckerQuoted 7 lines · Frederic Weisbecker
Buggy MWAIT implementations can't carry the
CPUIDLE_FLAG_MWAITflag because they require IPIs to wake up. Therefore they shouldn't be called withTIF_NR_POLLING.Reported-by Rafael J. Wysocki <[email protected]>Signed-off-by Frederic Weisbecker <[email protected]>
I would do a patch introducing arch_cpuidle_mwait_needs_ipi() or equivalent before patch [2/6] and I would use it in patch [2/6] right away.
Quoted 126 lines
--- arch/arm/include/asm/cpuidle.h | 2 ++ arch/arm64/include/asm/cpuidle.h | 3 +++ arch/powerpc/include/asm/cpuidle.h | 4 ++++ arch/riscv/include/asm/cpuidle.h | 2 ++ arch/x86/include/asm/cpuidle.h | 12 ++++++++++++ drivers/acpi/processor_idle.c | 4 +++- drivers/idle/intel_idle.c | 9 +++++++-- include/asm-generic/Kbuild | 1 + include/asm-generic/cpuidle.h | 10 ++++++++++ 9 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 arch/x86/include/asm/cpuidle.h create mode 100644 include/asm-generic/cpuidle.harch/arm/include/asm/cpuidle.h +2
-0@@ -55,4 +55,6 @@ struct arm_cpuidle_irq_context { };#define arm_cpuidle_save_irq_context (c) (void)c#define arm_cpuidle_restore_irq_context (c) (void)c +#include <asm-generic/cpuidle.h> +#endif arch/arm64/include/asm/cpuidle.h +3
-0@@ -38,4 +38,7 @@ struct arm_cpuidle_irq_context { };#define arm_cpuidle_save_irq_context (c) (void)c#define arm_cpuidle_restore_irq_context (c) (void)c#endif + +#include <asm-generic/cpuidle.h> +#endif arch/powerpc/include/asm/cpuidle.h +4
-0@@ -102,4 +102,8 @@ static inline void report_invalid_psscr_val(u64 psscr_val, int err)#endif +#ifndef __ASSEMBLY__ +#include <asm-generic/cpuidle.h> +#endif +#endif arch/riscv/include/asm/cpuidle.h +2
-0@@ -21,4 +21,6 @@ static inline void cpu_do_idle(void) wait_for_interrupt(); } +#include <asm-generic/cpuidle.h> +#endif arch/x86/include/asm/cpuidle.h new file +12
-0@@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_CPUIDLE_H +#define _ASM_X86_CPUIDLE_H + +#include <asm/cpufeature.h> + +static inline bool arch_cpuidle_mwait_needs_ipi (void ) +{ +return boot_cpu_has_bug (X86_BUG_MONITOR ); +} + +#endif /* _ASM_X86_CPUIDLE_H */ drivers/acpi/processor_idle.c +3
-1@@ -23,6 +23,7 @@#include <linux/perf_event.h> #include <acpi/processor.h> #include <linux/context_tracking.h> +#include <asm/cpuidle.h> /* * Include the apic definitions for x86 to have the APIC timer related defines @@ -806,7 +807,8 @@ static int acpi_processor_setup_cstates(struct acpi_processor *pr)if (cx->type ==ACPI_STATE_C1 || cx->type ==ACPI_STATE_C2 ) drv->safe_state_index = count; -if (cx->entry_method ==ACPI_CSTATE_FFH ) +if (cx->entry_method ==ACPI_CSTATE_FFH && + !arch_cpuidle_mwait_needs_ipi ()) state->flags |=CPUIDLE_FLAG_MWAIT ;/* drivers/idle/intel_idle.c +5
-1@@ -56,6 +56,7 @@#include <asm/mwait.h> #include <asm/spec-ctrl.h> #include <asm/fpu/api.h> +#include <asm/cpuidle.h> #define INTEL_IDLE_VERSION "0.5.1"@@ -1787,7 +1788,10 @@ static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv)if (cx->type >ACPI_STATE_C1 ) state->target_residency *=3 ; - state->flags =MWAIT2flg (cx->address) |CPUIDLE_FLAG_MWAIT ; + state->flags =MWAIT2flg (cx->address); + +if (!arch_cpuidle_mwait_needs_ipi ()) + state->flags |=CPUIDLE_FLAG_MWAIT ;if (cx->type >ACPI_STATE_C2 ) state->flags |=CPUIDLE_FLAG_TLB_FLUSHED ;
drivers/idle/intel_idle.c static bool __init intel_idle_verify_cstate(unsigned int mwait_hint) · line 2081
Quoted 6 lines
drivers/idle/intel_idle.c +2
-1@@ -2073,7 +2077,8 @@ static bool __init intel_idle_verify_cstate(unsigned int mwait_hint)static void state_update_enter_method (struct cpuidle_state *state,int cstate) { - state->flags |=CPUIDLE_FLAG_MWAIT ; +if (!arch_cpuidle_mwait_needs_ipi ()) + state->flags |=CPUIDLE_FLAG_MWAIT ;
Also, some code duplication could be avoided by having something like arch_x86_mwait_state() returning the flag if the condition is met:
state->flags |= arch_x86_mwait_state();
Quoted 35 lines
drivers/idle/intel_idle.c +0
-0if (state->flags &CPUIDLE_FLAG_INIT_XSTATE ) {/* include/asm-generic/Kbuild +1
-0@@ -13,6 +13,7 @@ mandatory-y += cacheflush.h mandatory-y += cfi.h mandatory-y += checksum.h mandatory-y += compat.h +mandatory-y += cpuidle.h mandatory-y += current.h mandatory-y += delay.h mandatory-y += device.hinclude/asm-generic/cpuidle.h new file +10
-1@@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_CPUIDLE_H +#define __ASM_CPUIDLE_H + +static inline bool arch_cpuidle_mwait_needs_ipi (void ) +{ +return true; +} + +#endif /* __ASM_CPUIDLE_H */ --2.46.0
Rafael J. Wysocki
maintainerRe: [PATCH 2/6] cpuidle: Introduce CPUIDLE_FLAG_MWAIT
In reply to Sudeep HollaQuoted 49 lines · Sudeep Holla
Quoted 47 lines · Rafael J. Wysocki
Quoted 30 lines · Frederic Weisbecker
From: Peter Zijlstra <[email protected]>
Provide a way to tell the cpuidle core about states monitoring
TIF_NEED_RESCHEDon the hardware level, monitor/mwait users being the only examples in use.This will allow cpuidle core to manage
TIF_NR_POLLINGon behalf of all kinds ofTIF_NEED_RESCHEDwatching states while keeping a necessary distinction for the governors between software loops polling onTIF_NEED_RESCHEDand hardware monitored writes to thread flags.[fweisbec: _ Initialize flag from
acpi_processor_setup_cstates()instead
ofacpi_processor_setup_lpi_states(), as the latter seem to be about arm64...
_ RenameCPUIDLE_FLAG_NO_IPItoCPUIDLE_FLAG_MWAIT]Signed-off-by Peter Zijlstra <[email protected]>Signed-off-by Frederic Weisbecker <[email protected]>--- drivers/acpi/processor_idle.c | 3 +++ drivers/idle/intel_idle.c | 5 ++++- include/linux/cpuidle.h | 1 + 3 files changed, 8 insertions(+), 1 deletion(-)drivers/acpi/processor_idle.c +0
-0drivers/acpi/processor_idle.c static int acpi_processor_setup_cstates(struct acpi_processor *pr) · line 810
Quoted 5 lines
drivers/acpi/processor_idle.c +2
-0@@ -806,6 +806,9 @@ static int acpi_processor_setup_cstates(struct acpi_processor *pr)if (cx->type ==ACPI_STATE_C1 || cx->type ==ACPI_STATE_C2 ) drv->safe_state_index = count; +if (cx->entry_method ==ACPI_CSTATE_FFH ) + state->flags |=CPUIDLE_FLAG_MWAIT ;FFH need not mean MWAIT in principle.
FFH in
_CSTmeans MWAIT in practice because_CSTis used on x86 which implements FFH through MWAIT, but it would be good at least to put a comment here to explain that this code is only expected to run on x86.Or better still, add something like
acpi_arch_idle_state_flags(u8 entry_method)that will returnCPUIDLE_FLAG_MWAITforACPI_CSTATE_FFHand 0 otherwise and then do
Quoted 6 lines
Quoted 2 lines
state->flags |= acpi_arch_idle_state_flags(cx->entry_method);+1, was about to suggest the same. Though I am not aware of any Arm platforms using C-States(LPI was added to suit Arm requirements), it is better to keep the FFH definition arch specific.
Which will be consistent with this patch among other things:
Rafael J. Wysocki
maintainerRe: [PATCH 4/6] cpuidle: Handle TIF_NR_POLLING on behalf of CPUIDLE_FLAG_MWAIT states
In reply to Frederic WeisbeckerQuoted 100 lines · Frederic Weisbecker
From: Peter Zijlstra <[email protected]>
The current handling of
TIF_NR_POLLINGis a bit of a maze:1)
TIF_NR_POLLINGis set on idle entry (one atomic set)2) Once cpuidle has selected an appropriate state and the tick is evaluated and then possibly stopped,
TIF_NR_POLLINGis cleared (one RmW operation)2) The cpuidle state is then called with
TIF_NR_POLLINGcleared but if the state polls on (or monitors)need_resched()it sets againTIF_NR_POLLINGbefore sleeping and clears it on wake-up. Summary:
another pair of set/clear3) Set back
TIF_NR_POLLING(one atomic set)4) goto 2) if
need_resched()is not setAll those costly atomic operations, fully ordered RmW for some of them, could be avoided if the cpuidle core knew in advance if the target state polls on (or monitors)
need_resched(). If so,TIF_NR_POLLINGcould simply be set once upon entering the idle loop and cleared once after idle loop exit.Start dealing with that with handling
TIF_NR_POLLINGon behalf of mwait based states.[fweisbec: _ Handle broadcast properly
_ Ignoremwait_idle()as it can be used bydefault_idle_call()]Signed-off-by Peter Zijlstra <[email protected]>Signed-off-by Frederic Weisbecker <[email protected]>--- arch/x86/include/asm/mwait.h | 27 +++++++++++------------- drivers/cpuidle/cpuidle.c | 22 +++++++++++++++++++- include/linux/sched/idle.h | 7 ++++++- kernel/sched/idle.c | 40 +++++++++++++----------------------- 4 files changed, 53 insertions(+), 43 deletions(-)arch/x86/include/asm/mwait.h +12
-15@@ -116,25 +116,22 @@ static __always_inline void __sti_mwait(unsigned long eax, unsigned long ecx)*/ static __always_inline void mwait_idle_with_hints(unsigned long eax, unsigned long ecx) { -if (static_cpu_has_bug (X86_BUG_MONITOR ) || !current_set_polling_and_test ()) { -if (static_cpu_has_bug (X86_BUG_CLFLUSH_MONITOR )) { -mb (); -clflush ((void *)¤t_thread_info ()->flags); -mb (); - } +if (static_cpu_has_bug (X86_BUG_CLFLUSH_MONITOR )) { +mb (); +clflush ((void *)¤t_thread_info ()->flags); +mb (); + } -__monitor ((void *)¤t_thread_info ()->flags,0 ,0 ); +__monitor ((void *)¤t_thread_info ()->flags,0 ,0 ); -if (!need_resched ()) { -if (ecx &1 ) { -__mwait (eax, ecx); - }else { -__sti_mwait (eax, ecx); -raw_local_irq_disable (); - } +if (!need_resched ()) { +if (ecx &1 ) { +__mwait (eax, ecx); + }else { +__sti_mwait (eax, ecx); +raw_local_irq_disable (); } } -current_clr_polling (); }/* drivers/cpuidle/cpuidle.c +1
-1@@ -217,10 +217,10 @@ noinstr int cpuidle_enter_state(struct cpuidle_device *dev, int index) {int entered_state; -struct cpuidle_state *target_state = &drv->states[index];bool broadcast = !!(target_state->flags &CPUIDLE_FLAG_TIMER_STOP );ktime_t time_start, time_end; +bool polling;instrumentation_begin ();
drivers/cpuidle/cpuidle.c noinstr int cpuidle_enter_state(struct cpuidle_device *dev, · line 240
Quoted 4 lines
drivers/cpuidle/cpuidle.c +1
-0@@ -237,6 +237,23 @@ noinstr int cpuidle_enter_state(struct cpuidle_device *dev, broadcast = false; } + polling = target_state->flags &CPUIDLE_FLAG_MWAIT ;
Hmmm. What about "polling" states, like state 0 on all x86?
They also monitor need_resched() -see poll_idle().
Quoted 59 lines
drivers/cpuidle/cpuidle.c +19
-0+ +/* +* If the target state doesn't poll on need_resched(), this is +* the last check after which further TIF_NEED_RESCHED remote setting +* will involve an IPI. +*/ +if (!polling &¤t_clr_polling_and_test ()) { +if (broadcast) +tick_broadcast_exit (); + dev->last_residency_ns =0 ; +local_irq_enable (); +instrumentation_end (); +return -EBUSY ; + } + +if (target_state->flags &CPUIDLE_FLAG_TLB_FLUSHED )leave_mm ();@@ -336,6 +353,9 @@ noinstr int cpuidle_enter_state(struct cpuidle_device *dev, dev->states_usage[index].rejected++; } +if (!polling) +__current_set_polling (); +instrumentation_end ();return entered_state;include/linux/sched/idle.h +6
-1@@ -68,6 +68,8 @@ static __always_inline bool __must_check current_set_polling_and_test(void) static __always_inline bool __must_check current_clr_polling_and_test(void) { +bool ret; +__current_clr_polling ();/* @@ -76,7 +78,10 @@ static __always_inline bool __must_check current_clr_polling_and_test(void)*/ smp_mb__after_atomic (); - return unlikely(tif_need_resched()); + ret =unlikely (tif_need_resched ()); +if (ret) +__current_set_polling (); + return ret; }#else kernel/sched/idle.c +0
-0
kernel/sched/idle.c void __cpuidle default_idle_call(void) · line 117
Quoted 5 lines
kernel/sched/idle.c +1
-1@@ -114,12 +114,13 @@ void __cpuidle default_idle_call(void)stop_critical_timings ();ct_cpuidle_enter (); -arch_cpu_idle (); +arch_cpu_idle ();// XXX assumes !polling
Well, what if x86_idle is default_idle()? Say, somebody boots with IDLE_NOMWAIT?
Quoted 78 lines
kernel/sched/idle.c +13
-26ct_cpuidle_exit ();start_critical_timings ();trace_cpu_idle (PWR_EVENT_EXIT ,smp_processor_id ());cond_tick_broadcast_exit (); +__current_set_polling (); }local_irq_enable ();instrumentation_end ();@@ -128,31 +129,14 @@ void __cpuidle default_idle_call(void)static int call_cpuidle_s2idle (struct cpuidle_driver *drv,struct cpuidle_device *dev) { +int ret; +if (current_clr_polling_and_test ())return -EBUSY ; -return cpuidle_enter_s2idle (drv, dev); -} - -static int call_cpuidle (struct cpuidle_driver *drv,struct cpuidle_device *dev, -int next_state) -{ -/* -* The idle task must be scheduled, it is pointless to go to idle, just -* update no idle residency and return. -*/ -if (current_clr_polling_and_test ()) { - dev->last_residency_ns =0 ; -local_irq_enable (); -return -EBUSY ; - } - -/* -* Enter the idle state previously returned by the governor decision. -* This function will block until an interrupt occurs and will take -* care of re-enabling the local interrupts -*/ -return cpuidle_enter (drv, dev, next_state); + ret =cpuidle_enter_s2idle (drv, dev); +__current_set_polling (); +return ret; }/** @@ -213,7 +197,7 @@ static void cpuidle_idle_call(void)tick_nohz_idle_stop_tick (); next_state =cpuidle_find_deepest_state (drv, dev, max_latency_ns); -call_cpuidle (drv, dev, next_state); +cpuidle_enter (drv, dev, next_state); }else {bool stop_tick = true;@@ -227,7 +211,12 @@ static void cpuidle_idle_call(void)else tick_nohz_idle_retain_tick (); - entered_state =call_cpuidle (drv, dev, next_state); +/* +* Enter the idle state previously returned by the governor decision. +* This function will block until an interrupt occurs and will take +* care of re-enabling the local interrupts. +*/ + entered_state =cpuidle_enter (drv, dev, next_state);/* * Give the governor an opportunity to reflect on the outcome */ @@ -235,7 +224,6 @@ static void cpuidle_idle_call(void) } exit_idle: -__current_set_polling ();/* * It is up to the idle functions to re-enable local interrupts --
Re: [PATCH 5/6] cpuidle: Remove call_cpuidle_s2idle()
In reply to Frederic WeisbeckerQuoted 2 lines · Frederic Weisbecker
This middle call is unecessary, especially now that its counterpart
unnecessary (one 'n' is missing)
Quoted 4 lines
call_cpuidle()has been removed.Suggested-by Rafael J. Wysocki <[email protected]>Signed-off-by Frederic Weisbecker <[email protected]>
For the code change
Quoted 48 lines
--- kernel/sched/idle.c | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-)kernel/sched/idle.c +6
-18@@ -126,19 +126,6 @@ void __cpuidle default_idle_call(void)instrumentation_end (); } -static int call_cpuidle_s2idle (struct cpuidle_driver *drv, -struct cpuidle_device *dev) -{ -int ret; - -if (current_clr_polling_and_test ()) -return -EBUSY ; - - ret =cpuidle_enter_s2idle (drv, dev); -__current_set_polling (); -return ret; -} -/** * cpuidle_idle_call - the main idle function * @@ -184,10 +171,12 @@ static void cpuidle_idle_call(void)u64 max_latency_ns;if (idle_should_enter_s2idle ()) { - - entered_state =call_cpuidle_s2idle (drv, dev); -if (entered_state >0 ) - goto exit_idle; +if (!current_clr_polling_and_test ()) { + entered_state =cpuidle_enter_s2idle (drv, dev); +__current_set_polling (); +if (entered_state >0 ) + goto exit_idle; + } max_latency_ns =U64_MAX ; }else { --2.46.0
Re: [PATCH 6/6] cpuidle: Handle TIF_NR_POLLING on behalf of software polling idle states
In reply to Frederic WeisbeckerQuoted 9 lines · Frederic Weisbecker
Software polling idle states set again
TIF_NR_POLLINGand clear it upon exit. This involves error prone duplicated code and wasted cycles performing atomic operations, sometimes RmW fully ordered.To avoid this, benefit instead from the same generic
TIF_NR_POLLINGhandling that is currently in use for hardware monitoring states.Signed-off-by Frederic Weisbecker <[email protected]>
So this is the part I was wondering about when I was looking at patch [4/6].
The code changes below look good to me, so feel free to add
to this patch when ready.
Quoted 145 lines
--- drivers/cpuidle/cpuidle-powernv.c | 10 ---------- drivers/cpuidle/cpuidle-pseries.c | 11 ----------- drivers/cpuidle/cpuidle.c | 2 +- drivers/cpuidle/poll_state.c | 30 ++++++++++++------------------ 4 files changed, 13 insertions(+), 40 deletions(-)drivers/cpuidle/cpuidle-powernv.c +0
-10@@ -71,8 +71,6 @@ static int snooze_loop(struct cpuidle_device *dev, {u64 snooze_exit_time; -set_thread_flag (TIF_POLLING_NRFLAG ); -local_irq_enable (); snooze_exit_time =get_tb () +get_snooze_timeout (dev, drv, index);@@ -81,21 +79,13 @@ static int snooze_loop(struct cpuidle_device *dev,HMT_very_low ();while (!need_resched ()) {if (likely (snooze_timeout_en) &&get_tb () > snooze_exit_time) { -/* -* Task has not woken up but we are exiting the polling -* loop anyway. Require a barrier after polling is -* cleared to order subsequent test of need_resched(). -*/ -clear_thread_flag (TIF_POLLING_NRFLAG ); dev->poll_time_limit = true; -smp_mb ();break ; } }HMT_medium ();ppc64_runlatch_on (); -clear_thread_flag (TIF_POLLING_NRFLAG );local_irq_disable ();drivers/cpuidle/cpuidle-pseries.c +0
-11@@ -40,8 +40,6 @@ int snooze_loop(struct cpuidle_device *dev, struct cpuidle_driver *drv, {u64 snooze_exit_time; -set_thread_flag (TIF_POLLING_NRFLAG ); -pseries_idle_prolog ();raw_local_irq_enable (); snooze_exit_time =get_tb () + snooze_timeout;@@ -51,21 +49,12 @@ int snooze_loop(struct cpuidle_device *dev, struct cpuidle_driver *drv,HMT_low ();HMT_very_low ();if (likely (snooze_timeout_en) &&get_tb () > snooze_exit_time) { -/* -* Task has not woken up but we are exiting the polling -* loop anyway. Require a barrier after polling is -* cleared to order subsequent test of need_resched(). -*/ dev->poll_time_limit = true; -clear_thread_flag (TIF_POLLING_NRFLAG ); -smp_mb ();break ; } }HMT_medium (); -clear_thread_flag (TIF_POLLING_NRFLAG ); -raw_local_irq_disable ();pseries_idle_epilog ();drivers/cpuidle/cpuidle.c +1
-1@@ -237,7 +237,7 @@ noinstr int cpuidle_enter_state(struct cpuidle_device *dev, broadcast = false; } - polling = target_state->flags &CPUIDLE_FLAG_MWAIT ; + polling = target_state->flags & (CPUIDLE_FLAG_MWAIT |CPUIDLE_FLAG_POLLING );/* * If the target state doesn't poll on need_resched(), this is drivers/cpuidle/poll_state.c +12
-19@@ -13,35 +13,29 @@ static int __cpuidle poll_idle(struct cpuidle_device *dev,struct cpuidle_driver *drv,int index) { -u64 time_start; - - time_start =local_clock_noinstr (); +u64 time_start =local_clock_noinstr (); +unsigned int loop_count =0 ; +u64 limit; dev->poll_time_limit = false;raw_local_irq_enable (); -if (!current_set_polling_and_test ()) { -unsigned int loop_count =0 ; -u64 limit; - limit =cpuidle_poll_time (drv, dev); + limit =cpuidle_poll_time (drv, dev); -while (!need_resched ()) { -cpu_relax (); -if (loop_count++ <POLL_IDLE_RELAX_COUNT ) -continue ; +while (!need_resched ()) { +cpu_relax (); +if (loop_count++ <POLL_IDLE_RELAX_COUNT ) +continue ; - loop_count =0 ; -if (local_clock_noinstr () - time_start > limit) { - dev->poll_time_limit = true; -break ; - } + loop_count =0 ; +if (local_clock_noinstr () - time_start > limit) { + dev->poll_time_limit = true; +break ; } }raw_local_irq_disable (); -current_clr_polling (); -return index; } --2.46.0
Hello Frederic,
Quoted 4 lines · Frederic Weisbecker
The
TIF_NR_POLLINGhandling againstTIF_NEED_RESCHEDpolling/monitoring idle states (mwait and also software polling) is a bit messy, with quite some wasted cycles spent on useless atomic operations. This tries to consolidate this state handling from the cpuidle core.
I tested this series on my 3rd Generation EPYC System with ipistorm and I've not observed any regressions in IPI throughput to idle CPUs in TIF_POLLING mode. Additionally tested few microbenchmarks and there were no surprises there either. Feel free to add:
Signature · 2 lines
Thanks and Regards, Prateek
Quoted 40 lines
Changes since v2:
_ Handle buggy mwait implementations (thanks Rafael)
Frederic Weisbecker (4):
cpuidle: Remove unnecessarycurrent_clr_polling_and_test()from
haltpoll
x86/cpuidle: Move buggy mwait implementations away from
CPUIDLE_FLAG_MWAIT
cpuidle: Removecall_cpuidle_s2idle()
cpuidle: HandleTIF_NR_POLLINGon behalf of software polling idle
statesPeter Zijlstra (2):
cpuidle: IntroduceCPUIDLE_FLAG_MWAIT
cpuidle: HandleTIF_NR_POLLINGon behalf ofCPUIDLE_FLAG_MWAITstatesarch/arm/include/asm/cpuidle.h | 2 ++ arch/arm64/include/asm/cpuidle.h | 3 ++ arch/powerpc/include/asm/cpuidle.h | 4 +++ arch/riscv/include/asm/cpuidle.h | 2 ++ arch/x86/include/asm/cpuidle.h | 12 +++++++ arch/x86/include/asm/mwait.h | 27 +++++++-------- drivers/acpi/processor_idle.c | 5 +++ drivers/cpuidle/cpuidle-haltpoll.c | 3 -- drivers/cpuidle/cpuidle-powernv.c | 10 ------ drivers/cpuidle/cpuidle-pseries.c | 11 ------- drivers/cpuidle/cpuidle.c | 22 ++++++++++++- drivers/cpuidle/poll_state.c | 30 +++++++---------- drivers/idle/intel_idle.c | 8 +++++ include/asm-generic/Kbuild | 1 + include/asm-generic/cpuidle.h | 10 ++++++ include/linux/cpuidle.h | 1 + include/linux/sched/idle.h | 7 +++- kernel/sched/idle.c | 53 +++++++++--------------------- 18 files changed, 114 insertions(+), 97 deletions(-)create mode 100644
arch/x86/include/asm/cpuidle.hcreate mode 100644include/asm-generic/cpuidle.h
K Prateek Nayak
reviewerRe: [PATCH 4/6] cpuidle: Handle TIF_NR_POLLING on behalf of CPUIDLE_FLAG_MWAIT states
In reply to Frederic WeisbeckerHello Frederic,
Quoted 30 lines · Frederic Weisbecker
[..snip..]
kernel/sched/idle.c +4
-2@@ -114,12 +114,13 @@ void __cpuidle default_idle_call(void)stop_critical_timings ();ct_cpuidle_enter (); -arch_cpu_idle (); +arch_cpu_idle ();// XXX assumes !polling ct_cpuidle_exit ();start_critical_timings ();trace_cpu_idle (PWR_EVENT_EXIT ,smp_processor_id ());cond_tick_broadcast_exit (); +__current_set_polling (); }local_irq_enable ();instrumentation_end ();@@ -128,31 +129,14 @@ void __cpuidle default_idle_call(void)static int call_cpuidle_s2idle (struct cpuidle_driver *drv,struct cpuidle_device *dev) { +int ret; +if (current_clr_polling_and_test ())return -EBUSY ; -return cpuidle_enter_s2idle (drv, dev);
kernel/sched/idle.c void __cpuidle default_idle_call(void) · line 144
Quoted 10 lines
kernel/sched/idle.c +0
-10-} - -static int call_cpuidle (struct cpuidle_driver *drv,struct cpuidle_device *dev, -int next_state) -{ -/* -* The idle task must be scheduled, it is pointless to go to idle, just -* update no idle residency and return. -*/ -if (current_clr_polling_and_test ()) {
nit.
Previously, if TIF_NEED_RESCHED was set by this point, the CPU would bail out early before entering the idle state but with this change, I believe only at need_resched() in mwait_idle_with_hints() do we realize we have a pending IPI / task wakeup. Is this a concern?
In my testing with ipistorm I could not see any difference in IPI throughput to polling idle CPUs but a bailout before the entry method for need_resched() can save a few cycles.
Signature · 2 lines
Thanks and Regards, Prateek
Quoted 48 lines
kernel/sched/idle.c +10
-14- dev->last_residency_ns =0 ; -local_irq_enable (); -return -EBUSY ; - } - -/* -* Enter the idle state previously returned by the governor decision. -* This function will block until an interrupt occurs and will take -* care of re-enabling the local interrupts -*/ -return cpuidle_enter (drv, dev, next_state); + ret =cpuidle_enter_s2idle (drv, dev); +__current_set_polling (); +return ret; }/** @@ -213,7 +197,7 @@ static void cpuidle_idle_call(void)tick_nohz_idle_stop_tick (); next_state =cpuidle_find_deepest_state (drv, dev, max_latency_ns); -call_cpuidle (drv, dev, next_state); +cpuidle_enter (drv, dev, next_state); }else {bool stop_tick = true;@@ -227,7 +211,12 @@ static void cpuidle_idle_call (void)else tick_nohz_idle_retain_tick (); - entered_state =call_cpuidle (drv, dev, next_state); +/* +* Enter the idle state previously returned by the governor decision. +* This function will block until an interrupt occurs and will take +* care of re-enabling the local interrupts. +*/ + entered_state =cpuidle_enter (drv, dev, next_state);/* * Give the governor an opportunity to reflect on the outcome */ @@ -235,7 +224,6 @@ static void cpuidle_idle_call(void) } exit_idle: -__current_set_polling ();/* * It is up to the idle functions to re-enable local interrupts