posix-timers: avoid nohz_full= IPIs via task cpu masks
Subsystems (2)
cover posix-timers: avoid nohz_full= IPIs via task cpu masks
This patchset avoids IPIs to nohz_full= CPUs when the intersection between the set of nohz_full CPUs and task allowed cpus is null.
See individual patches for details.
PATCH 1/2 nohz: try to avoid IPI when configuring per-CPU posix timer
When enabling per-CPU posix timers, an IPI to nohz_full CPUs might be performed (to re-read the dependencies and possibly not re-enter nohz_full on a given CPU).
A common case is for applications that run on nohz_full= CPUs to not use POSIX timers (eg DPDK). This patch skips the IPI
in case the task allowed mask does not intersect with nohz_full= CPU mask, when going through tick_nohz_dep_set_signal.
This reduces interruptions to nohz_full= CPUs.
--- include/linux/tick.h | 11 +++++++---- kernel/time/posix-cpu-timers.c | 4 ++-- kernel/time/tick-sched.c | 27 +++++++++++++++++++++++++-- 3 files changed, 34 insertions(+), 8 deletions(-)
include/linux/tick.h +7 -4
@@ -207,7 +207,8 @@ extern void tick_nohz_dep_set_task(struc enum tick_dep_bits bit);extern void tick_nohz_dep_clear_task (struct task_struct *tsk,enum tick_dep_bits bit); -extern void tick_nohz_dep_set_signal (struct signal_struct *signal, +extern void tick_nohz_dep_set_signal (struct task_struct *tsk, +struct signal_struct *signal,enum tick_dep_bits bit);extern void tick_nohz_dep_clear_signal (struct signal_struct *signal,enum tick_dep_bits bit);
@@ -252,11 +253,12 @@ static inline void tick_dep_clear_task(sif (tick_nohz_full_enabled ())tick_nohz_dep_clear_task (tsk, bit); } -static inline void tick_dep_set_signal (struct signal_struct *signal, +static inline void tick_dep_set_signal (struct task_struct *tsk, +struct signal_struct *signal,enum tick_dep_bits bit) {if (tick_nohz_full_enabled ()) -tick_nohz_dep_set_signal (signal, bit); +tick_nohz_dep_set_signal (tsk, signal, bit); }static inline void tick_dep_clear_signal (struct signal_struct *signal,enum tick_dep_bits bit)
@@ -284,7 +286,8 @@ static inline void tick_dep_set_task(str enum tick_dep_bits bit) { }static inline void tick_dep_clear_task (struct task_struct *tsk,enum tick_dep_bits bit) { } -static inline void tick_dep_set_signal (struct signal_struct *signal, +static inline void tick_dep_set_signal (struct task_struct *tsk, +struct signal_struct *signal,enum tick_dep_bits bit) { }static inline void tick_dep_clear_signal (struct signal_struct *signal,enum tick_dep_bits bit) { }
kernel/time/posix-cpu-timers.c +2 -2
@@ -523,7 +523,7 @@ static void arm_timer(struct k_itimer *tif (CPUCLOCK_PERTHREAD (timer->it_clock))tick_dep_set_task (p,TICK_DEP_BIT_POSIX_TIMER );else -tick_dep_set_signal (p->signal,TICK_DEP_BIT_POSIX_TIMER ); +tick_dep_set_signal (p, p->signal,TICK_DEP_BIT_POSIX_TIMER ); }/*
@@ -1358,7 +1358,7 @@ void set_process_cpu_timer(struct task_sif (*newval < *nextevt) *nextevt = *newval; -tick_dep_set_signal (tsk->signal,TICK_DEP_BIT_POSIX_TIMER ); +tick_dep_set_signal (tsk, tsk->signal,TICK_DEP_BIT_POSIX_TIMER ); } static int do_cpu_nanosleep(const clockid_t which_clock, int flags,
kernel/time/tick-sched.c +25 -2
@@ -302,6 +302,27 @@ static void tick_nohz_dep_set_all(atomic }/* +* Set bit on nohz full dependency, kicking all cpus +* only if task can run on nohz full CPUs. +*/ +static void tick_nohz_dep_set_all_cond (struct task_struct *tsk, +atomic_t *dep, +enum tick_dep_bits bit) +{ +int prev; +unsigned long flags; + + prev =atomic_fetch_or (BIT (bit), dep); +if (prev) +return ; + +raw_spin_lock_irqsave (&tsk->pi_lock, flags); +if (cpumask_intersects (&tsk->cpus_mask, tick_nohz_full_mask)) +tick_nohz_full_kick_all (); +raw_spin_unlock_irqrestore (&tsk->pi_lock, flags); +} + +/* * Set a global tick dependency. Used by perf events that rely on freq and * by unstable clock. */
@@ -382,9 +403,11 @@ EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_ta* Set a per-taskgroup tick dependency. Posix CPU timers need this in order to elapse * per process timers. */ -void tick_nohz_dep_set_signal (struct signal_struct *sig,enum tick_dep_bits bit) +void tick_nohz_dep_set_signal (struct task_struct *tsk, +struct signal_struct *sig, +enum tick_dep_bits bit) { -tick_nohz_dep_set_all (&sig->tick_dep_mask, bit); +tick_nohz_dep_set_all_cond (tsk, &sig->tick_dep_mask, bit); }void tick_nohz_dep_clear_signal (struct signal_struct *sig,enum tick_dep_bits bit)
Frederic Weisbecker
maintainerRe: [patch 1/2] nohz: try to avoid IPI when configuring per-CPU posix timer
Quoted 12 lines · Marcelo Tosatti
When enabling per-CPU posix timers, an IPI to
nohz_fullCPUs might be performed (to re-read the dependencies and possibly not re-enternohz_fullon a given CPU).A common case is for applications that run on nohz_full= CPUs to not use POSIX timers (eg DPDK). This patch skips the IPI
in case the task allowed mask does not intersect with nohz_full= CPU mask, when going throughtick_nohz_dep_set_signal.This reduces interruptions to nohz_full= CPUs.
Signed-off-by Marcelo Tosatti <[email protected]>
[...]
Quoted 8 lines
/*
+ * Set bit on nohz full dependency, kicking all cpus
+ * only if task can run on nohz full CPUs.
+ */kernel/time/tick-sched.c +4
-0+static void tick_nohz_dep_set_all_cond (struct task_struct *tsk, +atomic_t *dep, +enum tick_dep_bits bit) +{
kernel/time/tick-sched.c static void tick_nohz_dep_set_all(atomic · line 321
Quoted 10 lines
kernel/time/tick-sched.c +10
-0+int prev; +unsigned long flags; + + prev =atomic_fetch_or (BIT (bit), dep); +if (prev) +return ; + +raw_spin_lock_irqsave (&tsk->pi_lock, flags); +if (cpumask_intersects (&tsk->cpus_mask, tick_nohz_full_mask)) +tick_nohz_full_kick_all ();
So that's for one task but what about the other threads in that process? We are setting the tick dependency on all tasks sharing that struct signal.
Thanks.
Quoted 3 lines
kernel/time/tick-sched.c +3
-0+raw_spin_unlock_irqrestore (&tsk->pi_lock, flags); +} +
Re: [patch 1/2] nohz: try to avoid IPI when configuring per-CPU posix timer
Quoted 28 lines · Frederic Weisbecker
Quoted 12 lines · Marcelo Tosatti
When enabling per-CPU posix timers, an IPI to
nohz_fullCPUs might be performed (to re-read the dependencies and possibly not re-enternohz_fullon a given CPU).A common case is for applications that run on nohz_full= CPUs to not use POSIX timers (eg DPDK). This patch skips the IPI
in case the task allowed mask does not intersect with nohz_full= CPU mask, when going throughtick_nohz_dep_set_signal.This reduces interruptions to nohz_full= CPUs.
Signed-off-by Marcelo Tosatti <[email protected]>[...]
Quoted 14 lines
/*
+ * Set bit on nohz full dependency, kicking all cpus
+ * only if task can run on nohz full CPUs.
+ */kernel/time/tick-sched.c +10
-0+static void tick_nohz_dep_set_all_cond (struct task_struct *tsk, +atomic_t *dep, +enum tick_dep_bits bit) +{ +int prev; +unsigned long flags; + + prev =atomic_fetch_or (BIT (bit), dep); +if (prev) +return ;
Quoted 8 lines
kernel/time/tick-sched.c static void tick_nohz_dep_set_all(atomic · line 321
Quoted 4 lines
kernel/time/tick-sched.c +4
-0+ +raw_spin_lock_irqsave (&tsk->pi_lock, flags); +if (cpumask_intersects (&tsk->cpus_mask, tick_nohz_full_mask)) +tick_nohz_full_kick_all ();So that's for one task but what about the other threads in that process? We are setting the tick dependency on all tasks sharing that struct signal.
Hi Frederic,
Yep, fixing in -v2, thanks.
PATCH 2/2 nohz: try to avoid IPI when setting tick dependency for task
In reply to Marcelo TosattiWhen enabling per-CPU posix timers, an IPI to nohz_full CPUs might be performed (to re-read the dependencies and possibly not re-enter nohz_full on a given CPU).
A common case is for applications that run on nohz_full= CPUs to not use POSIX timers (eg DPDK).
This patch optimizes tick_nohz_dep_set_task to avoid kicking all nohz_full= CPUs in case the task allowed mask does not intersect with nohz_full= CPU mask,
when going through tick_nohz_dep_set_task.
This reduces interruptions to nohz_full= CPUs.
--- kernel/time/tick-sched.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-)
kernel/time/tick-sched.c +7 -2
@@ -383,11 +383,16 @@ void tick_nohz_dep_set_task(struct task_tick_nohz_full_kick ();preempt_enable (); }else { +unsigned long flags; +/* * Some future tick_nohz_full_kick_task() -* should optimize this. +* should further optimize this. */ -tick_nohz_full_kick_all (); +raw_spin_lock_irqsave (&tsk->pi_lock, flags); +if (cpumask_intersects (&tsk->cpus_mask, tick_nohz_full_mask)) +tick_nohz_full_kick_all (); +raw_spin_unlock_irqrestore (&tsk->pi_lock, flags); } } }
Frederic Weisbecker
maintainerRe: [patch 2/2] nohz: try to avoid IPI when setting tick dependency for task
Quoted 33 lines · Marcelo Tosatti
When enabling per-CPU posix timers, an IPI to
nohz_fullCPUs might be performed (to re-read the dependencies and possibly not re-enternohz_fullon a given CPU).A common case is for applications that run on nohz_full= CPUs to not use POSIX timers (eg DPDK).
This patch optimizes
tick_nohz_dep_set_taskto avoid kicking all nohz_full= CPUs in case the task allowed mask does not intersect with nohz_full= CPU mask,
when going throughtick_nohz_dep_set_task.This reduces interruptions to nohz_full= CPUs.
--- kernel/time/tick-sched.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-)Index:
linux-2.6/kernel/time/tick-sched.c
===================================================================kernel/time/tick-sched.c +2
-1@@ -383,11 +383,16 @@ void tick_nohz_dep_set_task(struct task_tick_nohz_full_kick ();preempt_enable (); }else { +unsigned long flags; +/* * Some future tick_nohz_full_kick_task() -* should optimize this.
kernel/time/tick-sched.c void tick_nohz_dep_set_task(struct task_ · line 398
Quoted 10 lines
kernel/time/tick-sched.c +5
-1+* should further optimize this. */ -tick_nohz_full_kick_all (); +raw_spin_lock_irqsave (&tsk->pi_lock, flags); +if (cpumask_intersects (&tsk->cpus_mask, tick_nohz_full_mask)) +tick_nohz_full_kick_all (); +raw_spin_unlock_irqrestore (&tsk->pi_lock, flags); } } }
Not long ago, Peterz suggested that we simply do:
kernel/time/tick-sched.c +20 -11
@@ -357,17 +357,26 @@ EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_cpu);void tick_nohz_dep_set_task (struct task_struct *tsk,enum tick_dep_bits bit) {if (!atomic_fetch_or (BIT (bit), &tsk->tick_dep_mask)) { -if (tsk == current) { -preempt_disable (); -tick_nohz_full_kick (); -preempt_enable (); - }else { -/* -* Some future tick_nohz_full_kick_task() -* should optimize this. -*/ -tick_nohz_full_kick_all (); - } +int cpu =task_cpu (tsk); + +/* +* If the task concurrently migrates to another cpu, +* we guarantee it sees the new tick dependency upon +* schedule. +* +* set_task_cpu(p, cpu); +* STORE p->cpu = @cpu +* __schedule() (switch to task 'p') +* LOCK rq->lock +* smp_mb__after_spin_lock() STORE p->tick_dep_mask +* tick_nohz_task_switch() smp_mb() (atomic_fetch_or()) +* LOAD p->tick_dep_mask LOAD p->cpu +*/ + +preempt_disable (); +if (cpu_online (cpu)) +tick_nohz_full_kick_cpu (cpu); +preempt_enable (); } }EXPORT_SYMBOL_GPL (tick_nohz_dep_set_task);
Re: [patch 2/2] nohz: try to avoid IPI when setting tick dependency for task
Quoted 79 lines · Frederic Weisbecker
Quoted 33 lines · Marcelo Tosatti
When enabling per-CPU posix timers, an IPI to
nohz_fullCPUs might be performed (to re-read the dependencies and possibly not re-enternohz_fullon a given CPU).A common case is for applications that run on nohz_full= CPUs to not use POSIX timers (eg DPDK).
This patch optimizes
tick_nohz_dep_set_taskto avoid kicking all nohz_full= CPUs in case the task allowed mask does not intersect with nohz_full= CPU mask,
when going throughtick_nohz_dep_set_task.This reduces interruptions to nohz_full= CPUs.
--- kernel/time/tick-sched.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-)Index:
linux-2.6/kernel/time/tick-sched.c
===================================================================kernel/time/tick-sched.c +2
-1@@ -383,11 +383,16 @@ void tick_nohz_dep_set_task(struct task_tick_nohz_full_kick ();preempt_enable (); }else { +unsigned long flags; +/* * Some future tick_nohz_full_kick_task() -* should optimize this. kernel/time/tick-sched.c void tick_nohz_dep_set_task(struct task_ · line 398
Quoted 10 lines
kernel/time/tick-sched.c +5
-1+* should further optimize this. */ -tick_nohz_full_kick_all (); +raw_spin_lock_irqsave (&tsk->pi_lock, flags); +if (cpumask_intersects (&tsk->cpus_mask, tick_nohz_full_mask)) +tick_nohz_full_kick_all (); +raw_spin_unlock_irqrestore (&tsk->pi_lock, flags); } } }Not long ago, Peterz suggested that we simply do:
kernel/time/tick-sched.c +13
-11@@ -357,17 +357,26 @@ EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_cpu);void tick_nohz_dep_set_task (struct task_struct *tsk,enum tick_dep_bits bit) {if (!atomic_fetch_or (BIT (bit), &tsk->tick_dep_mask)) { -if (tsk == current) { -preempt_disable (); -tick_nohz_full_kick (); -preempt_enable (); - }else { -/* -* Some future tick_nohz_full_kick_task() -* should optimize this. -*/ -tick_nohz_full_kick_all (); - } +int cpu =task_cpu (tsk); + +/* +* If the task concurrently migrates to another cpu, +* we guarantee it sees the new tick dependency upon +* schedule. +* +* set_task_cpu(p, cpu); +* STORE p->cpu = @cpu +* __schedule() (switch to task 'p') +* LOCK rq->lock +* smp_mb__after_spin_lock() STORE p->tick_dep_mask +* tick_nohz_task_switch() smp_mb() (atomic_fetch_or())
kernel/time/tick-sched.c EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_cpu); · line 382
Quoted 10 lines
kernel/time/tick-sched.c +7
-0+* LOAD p->tick_dep_mask LOAD p->cpu +*/ + +preempt_disable (); +if (cpu_online (cpu)) +tick_nohz_full_kick_cpu (cpu); +preempt_enable (); } }EXPORT_SYMBOL_GPL (tick_nohz_dep_set_task);
This can also be used for the signal case... thanks.