cpu-timers patches
cover cpu-timers patches
Here are patches for cpu-timers I mainly wrote to not call thread_group_cputime() in fastpath_timer_check(), which we discuss in this thread:
http://thread.gmane.org/gmane.linux.kernel/909349/focus=912327
On the way I fix some other bugs (SIGEV_NONE timer, old increase value in timer_settime()) and do other cleanups and optimizations, patch 3 however increase time when sighand->siglock is held.
0001-cpu-timers-simplify-RLIMIT_CPU-handling.patch 0002-cpu-timers-cleanup-arm_timer.patch 0003-cpu-timers-return-true-old-increase-value-in-timer_.patch 0004-cpu-timers-change-SIGEV_NONE-timer-handling.patch 0005-cpu-timer-change-arm_timer-locking.patch 0006-cpu-timers-optimize-run_posix_cpu_timers.patch
Cheers
Stanislaw
RFC PATCH 1/6 cpu-timers: simplify RLIMIT_CPU handling
Let always set expiration (prof_exp) cache when settings itimers, POSIX 1.b timers and RLIMIT_CPU. Together with settings expiration cache in fork, this allow to remove RLIMIT_CPU != inf check in fastpath_timer_check() and do some other cleanups.
--- kernel/posix-cpu-timers.c | 74 ++++++++++++++++----------------------------- 1 files changed, 26 insertions(+), 48 deletions(-)
kernel/posix-cpu-timers.c +26 -48
@@ -11,19 +11,17 @@#include <trace/events/timer.h> /* -* Called after updating RLIMIT_CPU to set timer expiration if necessary. +* Called after updating RLIMIT_CPU to run cputimer and set expiration cache +* if necessary. Need siglock since cpu timers can work with cputime_expires +* as well. */ void update_rlimit_cpu (unsigned long rlim_new) {cputime_t cputime =secs_to_cputime (rlim_new); -struct signal_struct *const sig = current->signal; -if (cputime_eq (sig->it[CPUCLOCK_PROF ].expires, cputime_zero) || -cputime_gt (sig->it[CPUCLOCK_PROF ].expires, cputime)) { -spin_lock_irq (¤t->sighand->siglock); -set_process_cpu_timer (current,CPUCLOCK_PROF , &cputime,NULL ); -spin_unlock_irq (¤t->sighand->siglock); - } +spin_lock_irq (¤t->sighand->siglock); +set_process_cpu_timer (current,CPUCLOCK_PROF , &cputime,NULL ); +spin_unlock_irq (¤t->sighand->siglock); }static int check_clock (const clockid_t which_clock)
@@ -564,7 +562,6 @@ static void arm_timer(struct k_itimer *timer, union cpu_time_count now)struct list_head *head, *listpos;struct cpu_timer_list *const nt = &timer->it.cpu;struct cpu_timer_list *next; -unsigned long i; head = (CPUCLOCK_PERTHREAD (timer->it_clock) ? p->cpu_timers : p->signal->cpu_timers);
@@ -630,20 +627,11 @@ static void arm_timer(struct k_itimer *timer, union cpu_time_count now)default :BUG ();case CPUCLOCK_VIRT : -if (expires_le (sig->it[CPUCLOCK_VIRT ].expires, - exp->cpu)) -break ; - sig->cputime_expires.virt_exp = exp->cpu; -break ; +if (expires_gt (sig->cputime_expires.virt_exp, exp->cpu)) + sig->cputime_expires.virt_exp = exp->cpu;case CPUCLOCK_PROF : -if (expires_le (sig->it[CPUCLOCK_PROF ].expires, - exp->cpu)) -break ; - i = sig->rlim[RLIMIT_CPU ].rlim_cur; -if (i !=RLIM_INFINITY && - i <=cputime_to_secs (exp->cpu)) -break ; - sig->cputime_expires.prof_exp = exp->cpu; +if (expires_gt (sig->cputime_expires.prof_exp, exp->cpu)) + sig->cputime_expires.prof_exp = exp->cpu;break ;case CPUCLOCK_SCHED : sig->cputime_expires.sched_exp = exp->sched;
@@ -1382,7 +1370,7 @@ static inline int fastpath_timer_check(struct task_struct *tsk)return 1 ; } -return sig->rlim[RLIMIT_CPU ].rlim_cur !=RLIM_INFINITY ; +return 0 ; }/*
@@ -1448,21 +1436,23 @@ void run_posix_cpu_timers(struct task_struct *tsk) }/* -* Set one of the process-wide special case CPU timers. +* Set one of the process-wide special case CPU timers or RLIMIT_CPU. * The tsk->sighand->siglock must be held by the caller. -* The *newval argument is relative and we update it to be absolute, *oldval -* is absolute and we update it to be relative. */ void set_process_cpu_timer (struct task_struct *tsk,unsigned int clock_idx,cputime_t *newval,cputime_t *oldval) {union cpu_time_count now; -struct list_head *head;BUG_ON (clock_idx ==CPUCLOCK_SCHED );cpu_timer_sample_group (clock_idx, tsk, &now);if (oldval) { +/* +* We are setting itimer. The *newval argument is relative +* and we update it to be absolute, *oldval is absolute and +* we update it to be relative. +*/ if (!cputime_eq (*oldval, cputime_zero)) {if (cputime_le (*oldval, now.cpu)) {/* Just about to fire. */
@@ -1475,33 +1465,21 @@ void set_process_cpu_timer(struct task_struct *tsk, unsigned int clock_idx,if (cputime_eq (*newval, cputime_zero))return ; *newval =cputime_add (*newval, now.cpu); - -/* -* If the RLIMIT_CPU timer will expire before the -* ITIMER_PROF timer, we have nothing else to do. -*/ -if (tsk->signal->rlim[RLIMIT_CPU ].rlim_cur - <cputime_to_secs (*newval)) -return ; }/* -* Check whether there are any process timers already set to fire -* before this one. If so, we don't have anything more to do. +* Update expiration cache if we are the earliest timer +* or RLIMIT_CPU limits before any other timer expire. */ - head = &tsk->signal->cpu_timers[clock_idx]; -if (list_empty (head) || -cputime_ge (list_first_entry (head, - struct cpu_timer_list, entry)->expires.cpu, - *newval)) { -switch (clock_idx) { -case CPUCLOCK_PROF : +switch (clock_idx) { +case CPUCLOCK_PROF : +if (expires_gt (tsk->signal->cputime_expires.prof_exp, *newval)) tsk->signal->cputime_expires.prof_exp = *newval; -break ; -case CPUCLOCK_VIRT : +break ; +case CPUCLOCK_VIRT : +if (expires_gt (tsk->signal->cputime_expires.virt_exp, *newval)) tsk->signal->cputime_expires.virt_exp = *newval; -break ; - } +break ; } }
Signature · 1 line
1.6.2.5
RFC PATCH 2/6 cpu-timers: cleanup arm_timer()
Updating expiration cache is messed in arm_timer(), clean it up.
--- kernel/posix-cpu-timers.c | 104 +++++++++++++++------------------------------ 1 files changed, 35 insertions(+), 69 deletions(-)
kernel/posix-cpu-timers.c +35 -69
@@ -546,97 +546,63 @@ static inline int expires_gt(cputime_t expires, cputime_t new_exp)cputime_gt (expires, new_exp); } -static inline int expires_le (cputime_t expires,cputime_t new_exp) -{ -return !cputime_eq (expires, cputime_zero) && -cputime_le (expires, new_exp); -}/* * Insert the timer on the appropriate list before any timers that * expire later. This must be called with the tasklist_lock held * for reading, and interrupts disabled. */ -static void arm_timer (struct k_itimer *timer,union cpu_time_count now) +static void arm_timer (struct k_itimer *timer) {struct task_struct *p = timer->it.cpu.task;struct list_head *head, *listpos; +struct task_cputime *cputime_expires;struct cpu_timer_list *const nt = &timer->it.cpu;struct cpu_timer_list *next; - head = (CPUCLOCK_PERTHREAD (timer->it_clock) ? - p->cpu_timers : p->signal->cpu_timers); +if (CPUCLOCK_PERTHREAD (timer->it_clock)) { + head = p->cpu_timers; + cputime_expires = &p->cputime_expires; + }else { + head = p->signal->cpu_timers; + cputime_expires = &p->signal->cputime_expires; + } head +=CPUCLOCK_WHICH (timer->it_clock);BUG_ON (!irqs_disabled ());spin_lock (&p->sighand->siglock); listpos = head; -if (CPUCLOCK_WHICH (timer->it_clock) ==CPUCLOCK_SCHED ) { -list_for_each_entry (next, head, entry) { -if (next->expires.sched > nt->expires.sched) -break ; - listpos = &next->entry; - } - }else { -list_for_each_entry (next, head, entry) { -if (cputime_gt (next->expires.cpu, nt->expires.cpu)) -break ; - listpos = &next->entry; - } +list_for_each_entry (next, head, entry) { +if (cpu_time_before (timer->it_clock, nt->expires, next->expires)) +break ; + listpos = &next->entry; }list_add (&nt->entry, listpos);if (listpos == head) { +union cpu_time_count *exp = &nt->expires; +/* -* We are the new earliest-expiring timer. -* If we are a thread timer, there can always -* be a process timer telling us to stop earlier. +* We are the new earliest-expiring POSIX 1.b timer, hence +* need to update expiration cache. Take into account that +* for process timers we share expiration cache with itimers +* and RLIMIT_CPU and for thread timers with RLIMIT_RTTIME. */ -if (CPUCLOCK_PERTHREAD (timer->it_clock)) { -union cpu_time_count *exp = &nt->expires; - -switch (CPUCLOCK_WHICH (timer->it_clock)) { -default : -BUG (); -case CPUCLOCK_PROF : -if (expires_gt (p->cputime_expires.prof_exp, - exp->cpu)) - p->cputime_expires.prof_exp = exp->cpu; -break ; -case CPUCLOCK_VIRT : -if (expires_gt (p->cputime_expires.virt_exp, - exp->cpu)) - p->cputime_expires.virt_exp = exp->cpu; -break ; -case CPUCLOCK_SCHED : -if (p->cputime_expires.sched_exp ==0 || - p->cputime_expires.sched_exp > exp->sched) - p->cputime_expires.sched_exp = - exp->sched; -break ; - } - }else { -struct signal_struct *const sig = p->signal; -union cpu_time_count *exp = &timer->it.cpu.expires; - -/* -* For a process timer, set the cached expiration time. -*/ -switch (CPUCLOCK_WHICH (timer->it_clock)) { -default : -BUG (); -case CPUCLOCK_VIRT : -if (expires_gt (sig->cputime_expires.virt_exp, exp->cpu)) - sig->cputime_expires.virt_exp = exp->cpu; -case CPUCLOCK_PROF : -if (expires_gt (sig->cputime_expires.prof_exp, exp->cpu)) - sig->cputime_expires.prof_exp = exp->cpu; -break ; -case CPUCLOCK_SCHED : - sig->cputime_expires.sched_exp = exp->sched; -break ; - } +switch (CPUCLOCK_WHICH (timer->it_clock)) { +case CPUCLOCK_PROF : +if (expires_gt (cputime_expires->prof_exp, exp->cpu)) + cputime_expires->prof_exp = exp->cpu; +break ; +case CPUCLOCK_VIRT : +if (expires_gt (cputime_expires->virt_exp, exp->cpu)) + cputime_expires->virt_exp = exp->cpu; +break ; +case CPUCLOCK_SCHED : +if (cputime_expires->sched_exp ==0 || + cputime_expires->sched_exp > exp->sched) + cputime_expires->sched_exp = exp->sched; +break ; } }
@@ -818,7 +784,7 @@ int posix_cpu_timer_set(struct k_itimer *timer, int flags,if (new_expires.sched !=0 && (timer->it_sigev_notify & ~SIGEV_THREAD_ID ) !=SIGEV_NONE &&cpu_time_before (timer->it_clock, val, new_expires)) { -arm_timer (timer, val); +arm_timer (timer); }read_unlock (&tasklist_lock);
@@ -1278,7 +1244,7 @@ void posix_cpu_timer_schedule(struct k_itimer *timer)/* * Now re-arm for the new expiry time. */ -arm_timer (timer, now); +arm_timer (timer); out_unlock:read_unlock (&tasklist_lock);
Signature · 1 line
1.6.2.5
RFC PATCH 3/6 cpu-timers: return true old increase value in timer_settime()
--- kernel/posix-cpu-timers.c | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-)
kernel/posix-cpu-timers.c +3 -2
@@ -675,7 +675,7 @@ int posix_cpu_timer_set(struct k_itimer *timer, int flags, struct itimerspec *new, struct itimerspec *old) { struct task_struct *p = timer->it.cpu.task; -union cpu_time_count old_expires, new_expires, val; +union cpu_time_count old_expires, new_expires, old_incr, val;int ret;if (unlikely (p ==NULL )) {
@@ -706,6 +706,7 @@ int posix_cpu_timer_set(struct k_itimer *timer, int flags,BUG_ON (!irqs_disabled ()); ret =0 ; + old_incr = timer->it.cpu.incr;spin_lock (&p->sighand->siglock); old_expires = timer->it.cpu.expires;if (unlikely (timer->it.cpu.firing)) {
@@ -821,7 +822,7 @@ int posix_cpu_timer_set(struct k_itimer *timer, int flags, out:if (old) {sample_to_timespec (timer->it_clock, - timer->it.cpu.incr, &old->it_interval); + old_incr, &old->it_interval); }return ret; }
Signature · 1 line
1.6.2.5
RFC PATCH 4/6 cpu-timers: change SIGEV_NONE timer handling
Currently when user setup timer without associated signal and process not use any other cpu timers, we are running tsk->signal->cputimer forewer, change that.
--- kernel/posix-cpu-timers.c | 28 ++++++---------------------- 1 files changed, 6 insertions(+), 22 deletions(-)
kernel/posix-cpu-timers.c +6 -22
@@ -614,7 +614,12 @@ static void arm_timer(struct k_itimer *timer)*/ static void cpu_timer_fire (struct k_itimer *timer) { -if (unlikely (timer->sigq ==NULL )) { +if ((timer->it_sigev_notify & ~SIGEV_THREAD_ID ) ==SIGEV_NONE ) { +/* +* User don't want any signal. +*/ + timer->it.cpu.expires.sched =0 ; + }else if (unlikely (timer->sigq ==NULL )) {/* * This a special case for clock_nanosleep, * not a normal timer from sys_timer_create.
@@ -783,7 +788,6 @@ int posix_cpu_timer_set(struct k_itimer *timer, int flags,*/ timer->it.cpu.expires = new_expires;if (new_expires.sched !=0 && - (timer->it_sigev_notify & ~SIGEV_THREAD_ID ) !=SIGEV_NONE &&cpu_time_before (timer->it_clock, val, new_expires)) {arm_timer (timer); }
@@ -808,7 +812,6 @@ int posix_cpu_timer_set(struct k_itimer *timer, int flags, timer->it_overrun =-1 ;if (new_expires.sched !=0 && - (timer->it_sigev_notify & ~SIGEV_THREAD_ID ) !=SIGEV_NONE && !cpu_time_before (timer->it_clock, val, new_expires)) {/* * The designated time already passed, so we notify
@@ -882,25 +885,6 @@ void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec *itp)read_unlock (&tasklist_lock); } -if ((timer->it_sigev_notify & ~SIGEV_THREAD_ID ) ==SIGEV_NONE ) { -if (timer->it.cpu.incr.sched ==0 && -cpu_time_before (timer->it_clock, - timer->it.cpu.expires, now)) { -/* -* Do-nothing timer expired and has no reload, -* so it's as if it was never set. -*/ - timer->it.cpu.expires.sched =0 ; - itp->it_value.tv_sec = itp->it_value.tv_nsec =0 ; -return ; - } -/* -* Account for any expirations and reloads that should -* have happened. -*/ -bump_cpu_timer (timer, now); - } -if (unlikely (clear_dead)) {/* * We've noticed that the thread is dead, but
Signature · 1 line
1.6.2.5
RFC PATCH 5/6 cpu-timers: change arm_timer() locking
Spread p->sighand->siglock locking scope to assure signal->cputimer is always running when we are setuping signal->cputime_expires.
This further assure we do not iterate over all threads in fastpath_timer_check() when calling thread_group_cputimer().
--- kernel/posix-cpu-timers.c | 14 +++++++------- 1 files changed, 7 insertions(+), 7 deletions(-)
kernel/posix-cpu-timers.c +7 -7
@@ -549,7 +549,7 @@ static inline int expires_gt(cputime_t expires, cputime_t new_exp)/* * Insert the timer on the appropriate list before any timers that * expire later. This must be called with the tasklist_lock held -* for reading, and interrupts disabled. +* for reading, interrupts disabled and p->sighand->siglock taken. */ static void arm_timer (struct k_itimer *timer) {
@@ -568,9 +568,6 @@ static void arm_timer(struct k_itimer *timer) } head +=CPUCLOCK_WHICH (timer->it_clock); -BUG_ON (!irqs_disabled ()); -spin_lock (&p->sighand->siglock); - listpos = head;list_for_each_entry (next, head, entry) {if (cpu_time_before (timer->it_clock, nt->expires, next->expires))
@@ -605,8 +602,6 @@ static void arm_timer(struct k_itimer *timer)break ; } } - -spin_unlock (&p->sighand->siglock); }/*
@@ -719,7 +714,6 @@ int posix_cpu_timer_set(struct k_itimer *timer, int flags, ret =TIMER_RETRY ; }else list_del_init (&timer->it.cpu.entry); -spin_unlock (&p->sighand->siglock);/* * We need to sample the current value to convert the new
@@ -773,6 +767,7 @@ int posix_cpu_timer_set(struct k_itimer *timer, int flags,* disable this firing since we are already reporting * it as an overrun (thanks to bump_cpu_timer above). */ +spin_unlock (&p->sighand->siglock);read_unlock (&tasklist_lock); goto out; }
@@ -792,6 +787,7 @@ int posix_cpu_timer_set(struct k_itimer *timer, int flags,arm_timer (timer); } +spin_unlock (&p->sighand->siglock);read_unlock (&tasklist_lock);/*
@@ -1201,6 +1197,7 @@ void posix_cpu_timer_schedule(struct k_itimer *timer) goto out; }read_lock (&tasklist_lock);/* arm_timer needs it. */ +spin_lock (&p->sighand->siglock); }else {read_lock (&tasklist_lock);if (unlikely (p->signal ==NULL )) {
@@ -1221,6 +1218,7 @@ void posix_cpu_timer_schedule(struct k_itimer *timer)clear_dead_task (timer, now); goto out_unlock; } +spin_lock (&p->sighand->siglock);cpu_timer_sample_group (timer->it_clock, p, &now);bump_cpu_timer (timer, now);/* Leave the tasklist_lock locked for the call below. */
@@ -1229,7 +1227,9 @@ void posix_cpu_timer_schedule(struct k_itimer *timer)/* * Now re-arm for the new expiry time. */ +BUG_ON (!irqs_disabled ());arm_timer (timer); +spin_unlock (&p->sighand->siglock); out_unlock:read_unlock (&tasklist_lock);
Signature · 1 line
1.6.2.5
RFC PATCH 6/6 cpu-timers: optimize run_posix_cpu_timers()
We can optimize and simplify things taking into account signal->cputimer is always running when we have configured any process wide cpu timer.
In check_process_timers(), we don't have to check if new updated value of signal->cputime_expires is smaller, since we maintain new first expiration time ({prof,virt,sched}_expires) in code flow and all other writes to expiration cache are protected by sighand->siglock .
--- kernel/posix-cpu-timers.c | 81 ++++++++++++++++++--------------------------- 1 files changed, 32 insertions(+), 49 deletions(-)
kernel/posix-cpu-timers.c +32 -49
@@ -995,14 +995,11 @@ static void check_thread_timers(struct task_struct *tsk, } } -static void stop_process_timers (struct task_struct *tsk) +static void stop_process_timers (struct signal_struct *sig) { -struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; +struct thread_group_cputimer *cputimer = &sig->cputimer;unsigned long flags; -if (!cputimer->running) -return ; -spin_lock_irqsave (&cputimer->lock, flags); cputimer->running =0 ;spin_unlock_irqrestore (&cputimer->lock, flags);
@@ -1042,6 +1039,23 @@ static void check_cpu_itimer(struct task_struct *tsk, struct cpu_itimer *it, } } +/** +* task_cputime_zero - Check a task_cputime struct for all zero fields. +* +* @cputime: The struct to compare. +* +* Checks @cputime to see if all fields are zero. Returns true if all fields +* are zero, false if any field is nonzero. +*/ +static inline int task_cputime_zero (const struct task_cputime *cputime) +{ +if (cputime_eq (cputime->utime, cputime_zero) && +cputime_eq (cputime->stime, cputime_zero) && + cputime->sum_exec_runtime ==0 ) +return 1 ; +return 0 ; +} +/* * Check for any per-thread CPU timers that have fired and move them * off the tsk->*_timers list onto the firing list. Per-thread timers
@@ -1058,19 +1072,6 @@ static void check_process_timers(struct task_struct *tsk,struct task_cputime cputime;/* -* Don't sample the current process CPU clocks if there are no timers. -*/ -if (list_empty (&timers[CPUCLOCK_PROF ]) && -cputime_eq (sig->it[CPUCLOCK_PROF ].expires, cputime_zero) && - sig->rlim[RLIMIT_CPU ].rlim_cur ==RLIM_INFINITY && -list_empty (&timers[CPUCLOCK_VIRT ]) && -cputime_eq (sig->it[CPUCLOCK_VIRT ].expires, cputime_zero) && -list_empty (&timers[CPUCLOCK_SCHED ])) { -stop_process_timers (tsk); -return ; - } - -/* * Collect the current process totals. */ thread_group_cputimer (tsk, &cputime);
@@ -1157,18 +1158,12 @@ static void check_process_timers(struct task_struct *tsk, } } -if (!cputime_eq (prof_expires, cputime_zero) && - (cputime_eq (sig->cputime_expires.prof_exp, cputime_zero) || -cputime_gt (sig->cputime_expires.prof_exp, prof_expires))) - sig->cputime_expires.prof_exp = prof_expires; -if (!cputime_eq (virt_expires, cputime_zero) && - (cputime_eq (sig->cputime_expires.virt_exp, cputime_zero) || -cputime_gt (sig->cputime_expires.virt_exp, virt_expires))) - sig->cputime_expires.virt_exp = virt_expires; -if (sched_expires !=0 && - (sig->cputime_expires.sched_exp ==0 || - sig->cputime_expires.sched_exp > sched_expires)) - sig->cputime_expires.sched_exp = sched_expires; + sig->cputime_expires.prof_exp = prof_expires; + sig->cputime_expires.virt_exp = virt_expires; + sig->cputime_expires.sched_exp = sched_expires; + +if (task_cputime_zero (&sig->cputime_expires)) +stop_process_timers (sig); }/*
@@ -1241,23 +1236,6 @@ out: }/** -* task_cputime_zero - Check a task_cputime struct for all zero fields. -* -* @cputime: The struct to compare. -* -* Checks @cputime to see if all fields are zero. Returns true if all fields -* are zero, false if any field is nonzero. -*/ -static inline int task_cputime_zero (const struct task_cputime *cputime) -{ -if (cputime_eq (cputime->utime, cputime_zero) && -cputime_eq (cputime->stime, cputime_zero) && - cputime->sum_exec_runtime ==0 ) -return 1 ; -return 0 ; -} - -/** * task_cputime_expired - Compare two task_cputime entities. * * @sample: The task_cputime structure to be checked for expiration.
@@ -1313,7 +1291,7 @@ static inline int fastpath_timer_check(struct task_struct *tsk) } sig = tsk->signal; -if (!task_cputime_zero (&sig->cputime_expires)) { +if (sig->cputimer.running) {struct task_cputime group_sample;thread_group_cputimer (tsk, &group_sample);
@@ -1350,7 +1328,12 @@ void run_posix_cpu_timers(struct task_struct *tsk)* put them on the firing list. */ check_thread_timers (tsk, &firing); -check_process_timers (tsk, &firing); +/* +* If there are any active process wide timers (POSIX 1.b, itimers, +* RLIMIT_CPU) cputimer must be running. +*/ +if (tsk->signal->cputimer.running) +check_process_timers (tsk, &firing);/* * We must release these locks before taking any timer's lock.
Signature · 1 line
1.6.2.5