cache __next_timer_interrupt result
RFC PATCH 1/1 cache __next_timer_interrupt result
From: Martin Schwidefsky <[email protected]>
Each time a cpu goes to sleep on a NOHZ=y system the timer wheel is searched for the next timer interrupt. It can take quite a few cycles to find the next pending timer. This patch adds a field to tvec_base that caches the result of __next_timer_interrupt. The hit ratio is around 80% on my thinkpad under normal use, on a server I've seen hit ratios from 5% to 95% dependent on the workload.
--- kernel/timer.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-)
kernel/timer.c +23 -1
@@ -72,6 +72,7 @@ struct tvec_base {spinlock_t lock;struct timer_list *running_timer;unsigned long timer_jiffies; +unsigned long next_timer;struct tvec_root tv1;struct tvec tv2;struct tvec tv3;
@@ -622,6 +623,9 @@ __mod_timer(struct timer_list *timer, unif (timer_pending (timer)) {detach_timer (timer,0 ); +if (timer->expires == base->next_timer && + !tbase_get_deferrable (timer->base)) + base->next_timer = base->timer_jiffies; ret =1 ; }else {if (pending_only)
@@ -663,6 +667,9 @@ __mod_timer(struct timer_list *timer, un } timer->expires = expires; +if (timer->expires < base->next_timer && + !tbase_get_deferrable (timer->base)) + base->next_timer = timer->expires;internal_add_timer (base, timer); out_unlock:
@@ -781,6 +788,9 @@ void add_timer_on(struct timer_list *timspin_lock_irqsave (&base->lock, flags);timer_set_base (timer, base);debug_timer_activate (timer); +if (timer->expires < base->next_timer && + !tbase_get_deferrable (timer->base)) + base->next_timer = timer->expires;internal_add_timer (base, timer);/* * Check whether the other CPU is idle and needs to be
@@ -817,6 +827,9 @@ int del_timer(struct timer_list *timer) base =lock_timer_base (timer, &flags);if (timer_pending (timer)) {detach_timer (timer,1 ); +if (timer->expires == base->next_timer && + !tbase_get_deferrable (timer->base)) + base->next_timer = base->timer_jiffies; ret =1 ; }spin_unlock_irqrestore (&base->lock, flags);
@@ -850,6 +863,9 @@ int try_to_del_timer_sync(struct timer_l ret =0 ;if (timer_pending (timer)) {detach_timer (timer,1 ); +if (timer->expires == base->next_timer && + !tbase_get_deferrable (timer->base)) + base->next_timer = base->timer_jiffies; ret =1 ; } out:
@@ -1134,7 +1150,9 @@ unsigned long get_next_timer_interrupt(uunsigned long expires;spin_lock (&base->lock); - expires =__next_timer_interrupt (base); +if (base->next_timer <= base->timer_jiffies) + base->next_timer =__next_timer_interrupt (base); + expires = base->next_timer;spin_unlock (&base->lock);if (time_before_eq (expires, now))
@@ -1523,6 +1541,7 @@ static int __cpuinit init_timers_cpu(intINIT_LIST_HEAD (base->tv1.vec + j); base->timer_jiffies = jiffies; + base->next_timer = base->timer_jiffies;return 0 ; }
@@ -1535,6 +1554,9 @@ static void migrate_timer_list(struct tv timer = list_first_entry(head, struct timer_list, entry);detach_timer (timer,0 );timer_set_base (timer, new_base); +if (timer->expires < new_base->next_timer && + !tbase_get_deferrable (timer->base)) + new_base->next_timer = timer->expires;internal_add_timer (new_base, timer); } }
Quoted 7 lines · Martin Schwidefsky
From: Martin Schwidefsky <[email protected]>
Each time a cpu goes to sleep on a NOHZ=y system the timer wheel is searched for the next timer interrupt. It can take quite a few cycles to find the next pending timer. This patch adds a field to
tvec_basethat caches the result of__next_timer_interrupt. The hit ratio is around 80% on my thinkpad under normal use, on a server I've seen
Nice, I like it.
Quoted 1 line
hit ratios from 5% to 95% dependent on the workload.
Which workloads result in lower hit ratios ? Heavy networking ?
Thanks,
tglx
Quoted 11 lines · On Wed, 22 Jul 2009 16:38:18 +0200 (CEST) Thomas Gleixner <[email protected]>…
Quoted 7 lines · Martin Schwidefsky
From: Martin Schwidefsky <[email protected]>
Each time a cpu goes to sleep on a NOHZ=y system the timer wheel is searched for the next timer interrupt. It can take quite a few cycles to find the next pending timer. This patch adds a field to
tvec_basethat caches the result of__next_timer_interrupt. The hit ratio is around 80% on my thinkpad under normal use, on a server I've seenNice, I like it.
Thanks :-)
Quoted 3 lines
Quoted 1 line
hit ratios from 5% to 95% dependent on the workload.
Which workloads result in lower hit ratios ? Heavy networking ?
5% ping-pong packet over loopback between two cpus. So yes, networking.
Signature · 4 lines
blue skies, Martin. "Reality continues to ruin my life." - Calvin.
[tip:timers/core] timers: Cache __next_timer_interrupt result
In reply to Martin SchwidefskyCommit-ID: 91ff44bdb806a3d26436cc4f5e4816d1ea75b34b Gitweb: http://git.kernel.org/tip/91ff44bdb806a3d26436cc4f5e4816d1ea75b34b Author: Martin Schwidefsky <[email protected]> AuthorDate: Tue, 21 Jul 2009 20:25:05 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Tue, 4 Aug 2009 16:07:51 +0200
timers: Cache __next_timer_interrupt result
Each time a cpu goes to sleep on a NOHZ=y system the timer wheel is searched for the next timer interrupt. It can take quite a few cycles to find the next pending timer.
This patch adds a field to tvec_base that caches the result of __next_timer_interrupt.
The hit ratio is around 80% on my thinkpad under normal use, on a server I've seen hit ratios from 5% to 95% dependent on the workload.
LKML-Reference: <20090721202505.7d56a079@skybase>
--- kernel/timer.c | 24 +++++++++++++++++++++++- 1 files changed, 23 insertions(+), 1 deletions(-)
kernel/timer.c +23 -1
@@ -72,6 +72,7 @@ struct tvec_base {spinlock_t lock;struct timer_list *running_timer;unsigned long timer_jiffies; +unsigned long next_timer;struct tvec_root tv1;struct tvec tv2;struct tvec tv3;
@@ -622,6 +623,9 @@ __mod_timer(struct timer_list *timer, unsigned long expires,if (timer_pending (timer)) {detach_timer (timer,0 ); +if (timer->expires == base->next_timer && + !tbase_get_deferrable (timer->base)) + base->next_timer = base->timer_jiffies; ret =1 ; }else {if (pending_only)
@@ -663,6 +667,9 @@ __mod_timer(struct timer_list *timer, unsigned long expires, } timer->expires = expires; +if (timer->expires < base->next_timer && + !tbase_get_deferrable (timer->base)) + base->next_timer = timer->expires;internal_add_timer (base, timer); out_unlock:
@@ -781,6 +788,9 @@ void add_timer_on(struct timer_list *timer, int cpu)spin_lock_irqsave (&base->lock, flags);timer_set_base (timer, base);debug_timer_activate (timer); +if (timer->expires < base->next_timer && + !tbase_get_deferrable (timer->base)) + base->next_timer = timer->expires;internal_add_timer (base, timer);/* * Check whether the other CPU is idle and needs to be
@@ -817,6 +827,9 @@ int del_timer(struct timer_list *timer) base =lock_timer_base (timer, &flags);if (timer_pending (timer)) {detach_timer (timer,1 ); +if (timer->expires == base->next_timer && + !tbase_get_deferrable (timer->base)) + base->next_timer = base->timer_jiffies; ret =1 ; }spin_unlock_irqrestore (&base->lock, flags);
@@ -850,6 +863,9 @@ int try_to_del_timer_sync(struct timer_list *timer) ret =0 ;if (timer_pending (timer)) {detach_timer (timer,1 ); +if (timer->expires == base->next_timer && + !tbase_get_deferrable (timer->base)) + base->next_timer = base->timer_jiffies; ret =1 ; } out:
@@ -1134,7 +1150,9 @@ unsigned long get_next_timer_interrupt(unsigned long now)unsigned long expires;spin_lock (&base->lock); - expires =__next_timer_interrupt (base); +if (base->next_timer <= base->timer_jiffies) + base->next_timer =__next_timer_interrupt (base); + expires = base->next_timer;spin_unlock (&base->lock);if (time_before_eq (expires, now))
@@ -1523,6 +1541,7 @@ static int __cpuinit init_timers_cpu(int cpu)INIT_LIST_HEAD (base->tv1.vec + j); base->timer_jiffies = jiffies; + base->next_timer = base->timer_jiffies;return 0 ; }
@@ -1535,6 +1554,9 @@ static void migrate_timer_list(struct tvec_base *new_base, struct list_head *hea timer = list_first_entry(head, struct timer_list, entry);detach_timer (timer,0 );timer_set_base (timer, new_base); +if (timer->expires < new_base->next_timer && + !tbase_get_deferrable (timer->base)) + new_base->next_timer = timer->expires;internal_add_timer (new_base, timer); } }
Re: [tip:timers/core] timers: Cache __next_timer_interrupt result
Quoted 8 lines · On Tue, 4 Aug 2009 14:16:04 GMT tip-bot for Martin Schwidefsky…
Commit-ID: 91ff44bdb806a3d26436cc4f5e4816d1ea75b34b Gitweb: http://git.kernel.org/tip/91ff44bdb806a3d26436cc4f5e4816d1ea75b34b Author: Martin Schwidefsky <[email protected]> AuthorDate: Tue, 21 Jul 2009 20:25:05 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Tue, 4 Aug 2009 16:07:51 +0200timers: Cache
__next_timer_interruptresult
Seeing that patch again after a few days and all of a sudden I find the bugs .. I really should use time_before and time_before_eq instead of comparing the expires values directly. New patch:
--
Subject: [PATCH] cache __next_timer_interrupt result
From: Martin Schwidefsky <[email protected]>
Each time a cpu goes to sleep on a NOHZ=y system the timer wheel is searched for the next timer interrupt. It can take quite a few cycles to find the next pending timer. This patch adds a field to tvec_base that caches the result of __next_timer_interrupt. The hit ratio is around 80% on my thinkpad under normal use, on a server I've seen hit ratios from 5% to 95% dependent on the workload.
---
kernel/timer.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-)
kernel/timer.c +23 -1
@@ -72,6 +72,7 @@ struct tvec_base {spinlock_t lock;struct timer_list *running_timer;unsigned long timer_jiffies; +unsigned long next_timer;struct tvec_root tv1;struct tvec tv2;struct tvec tv3;
@@ -622,6 +623,9 @@ __mod_timer(struct timer_list *timer, unif (timer_pending (timer)) {detach_timer (timer,0 ); +if (timer->expires == base->next_timer && + !tbase_get_deferrable (timer->base)) + base->next_timer = base->timer_jiffies; ret =1 ; }else {if (pending_only)
@@ -663,6 +667,9 @@ __mod_timer(struct timer_list *timer, un } timer->expires = expires; +if (time_before (timer->expires, base->next_timer) && + !tbase_get_deferrable (timer->base)) + base->next_timer = timer->expires;internal_add_timer (base, timer); out_unlock:
@@ -781,6 +788,9 @@ void add_timer_on(struct timer_list *timspin_lock_irqsave (&base->lock, flags);timer_set_base (timer, base);debug_timer_activate (timer); +if (time_before (timer->expires, base->next_timer) && + !tbase_get_deferrable (timer->base)) + base->next_timer = timer->expires;internal_add_timer (base, timer);/* * Check whether the other CPU is idle and needs to be
@@ -817,6 +827,9 @@ int del_timer(struct timer_list *timer) base =lock_timer_base (timer, &flags);if (timer_pending (timer)) {detach_timer (timer,1 ); +if (timer->expires == base->next_timer && + !tbase_get_deferrable (timer->base)) + base->next_timer = base->timer_jiffies; ret =1 ; }spin_unlock_irqrestore (&base->lock, flags);
@@ -850,6 +863,9 @@ int try_to_del_timer_sync(struct timer_l ret =0 ;if (timer_pending (timer)) {detach_timer (timer,1 ); +if (timer->expires == base->next_timer && + !tbase_get_deferrable (timer->base)) + base->next_timer = base->timer_jiffies; ret =1 ; } out:
@@ -1134,7 +1150,9 @@ unsigned long get_next_timer_interrupt(uunsigned long expires;spin_lock (&base->lock); - expires =__next_timer_interrupt (base); +if (time_before_eq (base->next_timer, base->timer_jiffies)) + base->next_timer =__next_timer_interrupt (base); + expires = base->next_timer;spin_unlock (&base->lock);if (time_before_eq (expires, now))
@@ -1523,6 +1541,7 @@ static int __cpuinit init_timers_cpu(intINIT_LIST_HEAD (base->tv1.vec + j); base->timer_jiffies = jiffies; + base->next_timer = base->timer_jiffies;return 0 ; }
@@ -1535,6 +1554,9 @@ static void migrate_timer_list(struct tv timer = list_first_entry(head, struct timer_list, entry);detach_timer (timer,0 );timer_set_base (timer, new_base); +if (time_before (timer->expires, new_base->next_timer) && + !tbase_get_deferrable (timer->base)) + new_base->next_timer = timer->expires;internal_add_timer (new_base, timer); } }
Signature · 4 lines
blue skies, Martin. "Reality continues to ruin my life." - Calvin.
[tip:timers/core] timers: Cache __next_timer_interrupt result
In reply to Martin SchwidefskyCommit-ID: 97fd9ed48ce2b807edc363bef3e817aeeb5cd5e6 Gitweb: http://git.kernel.org/tip/97fd9ed48ce2b807edc363bef3e817aeeb5cd5e6 Author: Martin Schwidefsky <[email protected]> AuthorDate: Tue, 21 Jul 2009 20:25:05 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Tue, 4 Aug 2009 20:28:25 +0200
timers: Cache __next_timer_interrupt result
Each time a cpu goes to sleep on a NOHZ=y system the timer wheel is searched for the next timer interrupt. It can take quite a few cycles to find the next pending timer.
This patch adds a field to tvec_base that caches the result of __next_timer_interrupt.
The hit ratio is around 80% on my thinkpad under normal use, on a server I've seen hit ratios from 5% to 95% dependent on the workload.
-v2: jiffies wrap fixes
LKML-Reference: <20090721202505.7d56a079@skybase>
--- kernel/timer.c | 24 +++++++++++++++++++++++- 1 files changed, 23 insertions(+), 1 deletions(-)
kernel/timer.c +23 -1
@@ -72,6 +72,7 @@ struct tvec_base {spinlock_t lock;struct timer_list *running_timer;unsigned long timer_jiffies; +unsigned long next_timer;struct tvec_root tv1;struct tvec tv2;struct tvec tv3;
@@ -622,6 +623,9 @@ __mod_timer(struct timer_list *timer, unsigned long expires,if (timer_pending (timer)) {detach_timer (timer,0 ); +if (timer->expires == base->next_timer && + !tbase_get_deferrable (timer->base)) + base->next_timer = base->timer_jiffies; ret =1 ; }else {if (pending_only)
@@ -663,6 +667,9 @@ __mod_timer(struct timer_list *timer, unsigned long expires, } timer->expires = expires; +if (time_before (timer->expires, base->next_timer) && + !tbase_get_deferrable (timer->base)) + base->next_timer = timer->expires;internal_add_timer (base, timer); out_unlock:
@@ -781,6 +788,9 @@ void add_timer_on(struct timer_list *timer, int cpu)spin_lock_irqsave (&base->lock, flags);timer_set_base (timer, base);debug_timer_activate (timer); +if (time_before (timer->expires, base->next_timer) && + !tbase_get_deferrable (timer->base)) + base->next_timer = timer->expires;internal_add_timer (base, timer);/* * Check whether the other CPU is idle and needs to be
@@ -817,6 +827,9 @@ int del_timer(struct timer_list *timer) base =lock_timer_base (timer, &flags);if (timer_pending (timer)) {detach_timer (timer,1 ); +if (timer->expires == base->next_timer && + !tbase_get_deferrable (timer->base)) + base->next_timer = base->timer_jiffies; ret =1 ; }spin_unlock_irqrestore (&base->lock, flags);
@@ -850,6 +863,9 @@ int try_to_del_timer_sync(struct timer_list *timer) ret =0 ;if (timer_pending (timer)) {detach_timer (timer,1 ); +if (timer->expires == base->next_timer && + !tbase_get_deferrable (timer->base)) + base->next_timer = base->timer_jiffies; ret =1 ; } out:
@@ -1134,7 +1150,9 @@ unsigned long get_next_timer_interrupt(unsigned long now)unsigned long expires;spin_lock (&base->lock); - expires =__next_timer_interrupt (base); +if (time_before_eq (base->next_timer, base->timer_jiffies)) + base->next_timer =__next_timer_interrupt (base); + expires = base->next_timer;spin_unlock (&base->lock);if (time_before_eq (expires, now))
@@ -1523,6 +1541,7 @@ static int __cpuinit init_timers_cpu(int cpu)INIT_LIST_HEAD (base->tv1.vec + j); base->timer_jiffies = jiffies; + base->next_timer = base->timer_jiffies;return 0 ; }
@@ -1535,6 +1554,9 @@ static void migrate_timer_list(struct tvec_base *new_base, struct list_head *hea timer = list_first_entry(head, struct timer_list, entry);detach_timer (timer,0 );timer_set_base (timer, new_base); +if (time_before (timer->expires, new_base->next_timer) && + !tbase_get_deferrable (timer->base)) + new_base->next_timer = timer->expires;internal_add_timer (new_base, timer); } }