sched/cputime: Protect some other sum_exec_runtime reads on 32 bit cpus
Subsystems (2)
PATCH v3 1/1 sched/cputime: Protect some other sum_exec_runtime reads on 32 bit cpus
We protect 64bit sum_exec_runtime variable against inconsistency on 32 bit architectures when reading it on thread_group_cputime(), but we also read it on other places not protected.
Patch changes that, except places where sum_exec_runtime is read for debuging purpose and where we also read some other u64 tsk->se.X or tsk->sched_info.X variables without protection like on kernel/delayacct.c, fs/proc/base.c and kernel/sched/debug.c . But since those are for debug or for printing statistics purpose and inconsistency is very unlike, we can ignore that.
Patch fixes places were inconsistency can results of cpuclock monotonicity breakage or cputimer expire too early or too late.
---
v2 -> v3:
- don't drop rq->lock
- don't use read_sum_exec_runtime() where we read other 64bit variables without protection for debug/statistics purpose
include/linux/sched.h | 9 +++++++++ kernel/exit.c | 2 +- kernel/sched/core.c | 18 ++++++++++++++++++ kernel/sched/cputime.c | 22 +--------------------- kernel/time/posix-cpu-timers.c | 4 ++-- 5 files changed, 31 insertions(+), 24 deletions(-)
include/linux/sched.h +9 -0
@@ -2508,6 +2508,15 @@ static inline void disable_sched_clock_irqtime(void) {}extern unsigned long long task_sched_runtime (struct task_struct *task); +#ifdef CONFIG_64BIT +static inline u64 read_sum_exec_runtime (struct task_struct *t) +{ +return t->se.sum_exec_runtime; +} +#else +extern u64 read_sum_exec_runtime (struct task_struct *t); +#endif +/* sched_exec is called by processes performing an exec */ #ifdef CONFIG_SMP extern void sched_exec (void );
kernel/exit.c +1 -1
@@ -134,7 +134,7 @@ static void __exit_signal(struct task_struct *tsk) sig->inblock +=task_io_get_inblock (tsk); sig->oublock +=task_io_get_oublock (tsk);task_io_accounting_add (&sig->ioac, &tsk->ioac); - sig->sum_sched_runtime += tsk->se.sum_exec_runtime; + sig->sum_sched_runtime +=read_sum_exec_runtime (tsk); sig->nr_threads--;__unhash_process (tsk, group_dead);write_sequnlock (&sig->stats_lock);
kernel/sched/core.c +18 -0
@@ -3034,6 +3034,24 @@ unsigned long long task_sched_runtime(struct task_struct *p)return ns; } +#ifndef CONFIG_64BIT +/* +* Protect 64bit read against concurrent write in update_curr(). +*/ +u64 read_sum_exec_runtime (struct task_struct *t) +{ +u64 ns; +struct rq_flags rf; +struct rq *rq; + + rq =task_rq_lock (t, &rf); + ns = t->se.sum_exec_runtime; +task_rq_unlock (rq, t, &rf); + +return ns; +} +#endif +/* * This function gets called by the timer code, with HZ frequency. * We call it with interrupts disabled.
kernel/sched/cputime.c +1 -21
@@ -306,26 +306,6 @@ static inline cputime_t account_other_time(cputime_t max)return accounted; } -#ifdef CONFIG_64BIT -static inline u64 read_sum_exec_runtime (struct task_struct *t) -{ -return t->se.sum_exec_runtime; -} -#else -static u64 read_sum_exec_runtime (struct task_struct *t) -{ -u64 ns; -struct rq_flags rf; -struct rq *rq; - - rq =task_rq_lock (t, &rf); - ns = t->se.sum_exec_runtime; -task_rq_unlock (rq, t, &rf); - -return ns; -} -#endif -/* * Accumulate raw cputime values of dead tasks (sig->[us]time) and live * tasks (sum on group iteration) belonging to @tsk's group.
@@ -702,7 +682,7 @@ out:void task_cputime_adjusted (struct task_struct *p,cputime_t *ut,cputime_t *st) {struct task_cputime cputime = { - .sum_exec_runtime = p->se.sum_exec_runtime, + .sum_exec_runtime =read_sum_exec_runtime (p), };task_cputime (p, &cputime.utime, &cputime.stime);
kernel/time/posix-cpu-timers.c +2 -2
@@ -848,7 +848,7 @@ static void check_thread_timers(struct task_struct *tsk, tsk_expires->virt_exp =expires_to_cputime (expires); tsk_expires->sched_exp =check_timers_list (++timers, firing, - tsk->se.sum_exec_runtime); +read_sum_exec_runtime (tsk));/* * Check for the special case thread timers.
@@ -1115,7 +1115,7 @@ static inline int fastpath_timer_check(struct task_struct *tsk)struct task_cputime task_sample;task_cputime (tsk, &task_sample.utime, &task_sample.stime); - task_sample.sum_exec_runtime = tsk->se.sum_exec_runtime; + task_sample.sum_exec_runtime =read_sum_exec_runtime (tsk);if (task_cputime_expired (&task_sample, &tsk->cputime_expires))return 1 ; }
Signature · 1 line
1.8.3.1
Peter Zijlstra
maintainerkernel/exit.c static void __exit_signal(struct task_struct *tsk) · line 140
Quoted 13 lines · Stanislaw Gruszka
kernel/exit.c +1
-1@@ -134,7 +134,7 @@ static void __exit_signal(struct task_struct *tsk) sig->inblock +=task_io_get_inblock (tsk); sig->oublock +=task_io_get_oublock (tsk);task_io_accounting_add (&sig->ioac, &tsk->ioac); - sig->sum_sched_runtime += tsk->se.sum_exec_runtime; + sig->sum_sched_runtime +=read_sum_exec_runtime (tsk); sig->nr_threads--;__unhash_process (tsk, group_dead);write_sequnlock (&sig->stats_lock);
If I'm not mistaken, at this point @p is dead, sum_exec_runtime will not ever be updated again.
kernel/sched/cputime.c out: · line 688
Quoted 13 lines
kernel/sched/cputime.c +1
-1@@ -702,7 +682,7 @@ out:void task_cputime_adjusted (struct task_struct *p,cputime_t *ut,cputime_t *st) {struct task_cputime cputime = { - .sum_exec_runtime = p->se.sum_exec_runtime, + .sum_exec_runtime =read_sum_exec_runtime (p), };task_cputime (p, &cputime.utime, &cputime.stime);
Right, even if @p == current, this can race with an interrupt (the tick) updating sum_exec_runtime.
kernel/time/posix-cpu-timers.c static void check_thread_timers(struct task_struct *tsk, · line 854
Quoted 13 lines
kernel/time/posix-cpu-timers.c +1
-1@@ -848,7 +848,7 @@ static void check_thread_timers(struct task_struct *tsk, tsk_expires->virt_exp =expires_to_cputime (expires); tsk_expires->sched_exp =check_timers_list (++timers, firing, - tsk->se.sum_exec_runtime); +read_sum_exec_runtime (tsk));/* * Check for the special case thread timers.
@tsk == current and IRQs are disabled, sum_exec_runtime cannot be updated.
kernel/time/posix-cpu-timers.c static inline int fastpath_timer_check(struct task_struct *tsk) · line 1121
Quoted 9 lines
kernel/time/posix-cpu-timers.c +1
-1@@ -1115,7 +1115,7 @@ static inline int fastpath_timer_check(struct task_struct *tsk)struct task_cputime task_sample;task_cputime (tsk, &task_sample.utime, &task_sample.stime); - task_sample.sum_exec_runtime = tsk->se.sum_exec_runtime; + task_sample.sum_exec_runtime =read_sum_exec_runtime (tsk);if (task_cputime_expired (&task_sample, &tsk->cputime_expires))return 1 ; }
Same.
Quoted 17 lines · Peter Zijlstra
kernel/exit.c static void __exit_signal(struct task_struct *tsk) · line 140
Quoted 13 lines · Stanislaw Gruszka
kernel/exit.c +1
-1@@ -134,7 +134,7 @@ static void __exit_signal(struct task_struct *tsk) sig->inblock +=task_io_get_inblock (tsk); sig->oublock +=task_io_get_oublock (tsk);task_io_accounting_add (&sig->ioac, &tsk->ioac); - sig->sum_sched_runtime += tsk->se.sum_exec_runtime; + sig->sum_sched_runtime +=read_sum_exec_runtime (tsk); sig->nr_threads--;__unhash_process (tsk, group_dead);write_sequnlock (&sig->stats_lock);If I'm not mistaken, at this point @p is dead,
sum_exec_runtimewill not ever be updated again.
I think at this point find_task_by_vpid() will still return not-null pointer, hence we can update sum_exec_runtime via:
posix_cpu_{clock_get(),timer_set(),timer_get()} -> cpu_clock_sample() -> task_sched_runtime() -> update_curr()
if some thread will do cpuclock/cputimer related syscall for different thread and CPUCLOCK_SCHED clock.
Quoted 11 lines
Quoted 11 lines
kernel/time/posix-cpu-timers.c +1
-1@@ -848,7 +848,7 @@ static void check_thread_timers(struct task_struct *tsk, tsk_expires->virt_exp =expires_to_cputime (expires); tsk_expires->sched_exp =check_timers_list (++timers, firing, - tsk->se.sum_exec_runtime); +read_sum_exec_runtime (tsk));
Quoted 5 lines
kernel/time/posix-cpu-timers.c static void check_thread_timers(struct task_struct *tsk, · line 854
Quoted 2 lines
kernel/time/posix-cpu-timers.c +0
-0/* * Check for the special case thread timers. @tsk == current and IRQs are disabled,
sum_exec_runtimecannot be updated.
It still can be updated via syscalls on different thread, same paths as above.
Quoted 11 lines
kernel/time/posix-cpu-timers.c static inline int fastpath_timer_check(struct task_struct *tsk) · line 1121
Quoted 9 lines
kernel/time/posix-cpu-timers.c +1
-1@@ -1115,7 +1115,7 @@ static inline int fastpath_timer_check(struct task_struct *tsk)struct task_cputime task_sample;task_cputime (tsk, &task_sample.utime, &task_sample.stime); - task_sample.sum_exec_runtime = tsk->se.sum_exec_runtime; + task_sample.sum_exec_runtime =read_sum_exec_runtime (tsk);if (task_cputime_expired (&task_sample, &tsk->cputime_expires))return 1 ; }Same.
Same :-) But this could be optimized to avoid taking lock if not needed:
if (expires->sum_exec_runtime != 0) task_sample.sum_exec_runtime = read_sum_exec_runtime(tsk);
Stanislaw
Peter Zijlstra
maintainerQuoted 19 lines · Stanislaw Gruszka
Quoted 17 lines · Peter Zijlstra
kernel/exit.c static void __exit_signal(struct task_struct *tsk) · line 140
Quoted 13 lines · Stanislaw Gruszka
kernel/exit.c +1
-1@@ -134,7 +134,7 @@ static void __exit_signal(struct task_struct *tsk) sig->inblock +=task_io_get_inblock (tsk); sig->oublock +=task_io_get_oublock (tsk);task_io_accounting_add (&sig->ioac, &tsk->ioac); - sig->sum_sched_runtime += tsk->se.sum_exec_runtime; + sig->sum_sched_runtime +=read_sum_exec_runtime (tsk); sig->nr_threads--;__unhash_process (tsk, group_dead);write_sequnlock (&sig->stats_lock);If I'm not mistaken, at this point @p is dead,
sum_exec_runtimewill not ever be updated again.
Quoted 8 lines
I think at this point
find_task_by_vpid()will still return not-null pointer, hence we can updatesum_exec_runtimevia:posix_cpu_{clock_get(),timer_set(),timer_get()} ->
cpu_clock_sample()->task_sched_runtime()->update_curr()if some thread will do cpuclock/cputimer related syscall for different thread and
CPUCLOCK_SCHEDclock.
Ah, ok. I always forget how the exit path works..
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
kernel/time/posix-cpu-timers.c +1
-1@@ -848,7 +848,7 @@ static void check_thread_timers(struct task_struct *tsk, tsk_expires->virt_exp =expires_to_cputime (expires); tsk_expires->sched_exp =check_timers_list (++timers, firing, - tsk->se.sum_exec_runtime); +read_sum_exec_runtime (tsk));
Quoted 8 lines
Quoted 5 lines
kernel/time/posix-cpu-timers.c static void check_thread_timers(struct task_struct *tsk, · line 854
Quoted 2 lines
kernel/time/posix-cpu-timers.c +0
-0/* * Check for the special case thread timers. @tsk == current and IRQs are disabled,
sum_exec_runtimecannot be updated.It still can be updated via syscalls on different thread, same paths as above.
Yep, realized that over-night.
Peter Zijlstra
maintainerQuoted 20 lines · Peter Zijlstra
kernel/time/posix-cpu-timers.c static void check_thread_timers(struct task_struct *tsk, · line 854
Quoted 13 lines
kernel/time/posix-cpu-timers.c +1
-1@@ -848,7 +848,7 @@ static void check_thread_timers(struct task_struct *tsk, tsk_expires->virt_exp =expires_to_cputime (expires); tsk_expires->sched_exp =check_timers_list (++timers, firing, - tsk->se.sum_exec_runtime); +read_sum_exec_runtime (tsk));/* * Check for the special case thread timers. @tsk == current and IRQs are disabled,
sum_exec_runtimecannot be updated.Quoted 3 lines
kernel/time/posix-cpu-timers.c +0
-0@@ -1115,7 +1115,7 @@ static inline int fastpath_timer_check(struct task_struct *tsk)struct task_cputime task_sample;
Quoted 8 lines
kernel/time/posix-cpu-timers.c static inline int fastpath_timer_check(struct task_struct *tsk) · line 1121
Quoted 6 lines
kernel/time/posix-cpu-timers.c +1
-1task_cputime (tsk, &task_sample.utime, &task_sample.stime); - task_sample.sum_exec_runtime = tsk->se.sum_exec_runtime; + task_sample.sum_exec_runtime =read_sum_exec_runtime (tsk);if (task_cputime_expired (&task_sample, &tsk->cputime_expires))return 1 ; }Same.
Sorry, I forgot syscalls, those can muck with update_curr() remotely. So yes, those are needed.