sched/cputime: irqtime cleanups
Subsystems (2)
cover sched/cputime: irqtime cleanups
This series contains a few optimizations against irq disabling, and the rest is consolidation.
git://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git
sched/irqtime
HEAD: 63024a0947091e0a20bafa33ccccb685ae33c275
Thanks,
Frederic
---
Frederic Weisbecker (5):
irqtime: No need for preempt-safe accessors
irqtime: Remove needless IRQs disablement on kcpustat update
u64_stats: Introduce IRQs disabled helpers
irqtime: Consolidate accounting synchronization with u64_stats API
irqtime: Consolidate irqtime flushing code
include/linux/u64_stats_sync.h | 49 +++++++++++++++++++++----------------- kernel/sched/cputime.c | 51 ++++++++++++++++------------------------ kernel/sched/sched.h | 53 ++++++++++++------------------------------ 3 files changed, 63 insertions(+), 90 deletions(-)
PATCH 1/5 irqtime: No need for preempt-safe accessors
We can safely use the preempt-unsafe accessors for irqtime when we flush its counters to kcpustat as IRQs are disabled at this time.
--- kernel/sched/cputime.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
kernel/sched/cputime.c +2 -2
@@ -82,7 +82,7 @@ static cputime_t irqtime_account_hi_update(cputime_t maxtime)cputime_t irq_cputime;local_irq_save (flags); - irq_cputime =nsecs_to_cputime64 (this_cpu_read (cpu_hardirq_time)) - + irq_cputime =nsecs_to_cputime64 (__this_cpu_read (cpu_hardirq_time)) - cpustat[CPUTIME_IRQ ]; irq_cputime =min (irq_cputime, maxtime); cpustat[CPUTIME_IRQ ] += irq_cputime;
@@ -97,7 +97,7 @@ static cputime_t irqtime_account_si_update(cputime_t maxtime)cputime_t softirq_cputime;local_irq_save (flags); - softirq_cputime =nsecs_to_cputime64 (this_cpu_read (cpu_softirq_time)) - + softirq_cputime =nsecs_to_cputime64 (__this_cpu_read (cpu_softirq_time)) - cpustat[CPUTIME_SOFTIRQ ]; softirq_cputime =min (softirq_cputime, maxtime); cpustat[CPUTIME_SOFTIRQ ] += softirq_cputime;
Signature · 1 line
2.7.0
Re: [PATCH 1/5] irqtime: No need for preempt-safe accessors
Quoted 12 lines · Frederic Weisbecker
We can safely use the preempt-unsafe accessors for irqtime when we flush its counters to kcpustat as IRQs are disabled at this time.
Cc Rik van Riel <[email protected]>Cc Paolo Bonzini <[email protected]>Cc Wanpeng Li <[email protected]>Cc Mike Galbraith <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Ingo Molnar <[email protected]>Cc Eric Dumazet <[email protected]>Signed-off-by Frederic Weisbecker <[email protected]>
Signature · 1 line
All Rights Reversed.
PATCH 2/5 irqtime: Remove needless IRQs disablement on kcpustat update
In reply to Frederic WeisbeckerThe callers of the functions performing irqtime kcpustat updates have IRQS disabled, no need to disable them again.
--- kernel/sched/cputime.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-)
kernel/sched/cputime.c +2 -6
@@ -78,30 +78,26 @@ EXPORT_SYMBOL_GPL(irqtime_account_irq);static cputime_t irqtime_account_hi_update (cputime_t maxtime) {u64 *cpustat = kcpustat_this_cpu->cpustat; -unsigned long flags;cputime_t irq_cputime; -local_irq_save (flags); irq_cputime =nsecs_to_cputime64 (__this_cpu_read (cpu_hardirq_time)) - cpustat[CPUTIME_IRQ ]; irq_cputime =min (irq_cputime, maxtime); cpustat[CPUTIME_IRQ ] += irq_cputime; -local_irq_restore (flags); +return irq_cputime; }static cputime_t irqtime_account_si_update (cputime_t maxtime) {u64 *cpustat = kcpustat_this_cpu->cpustat; -unsigned long flags;cputime_t softirq_cputime; -local_irq_save (flags); softirq_cputime =nsecs_to_cputime64 (__this_cpu_read (cpu_softirq_time)) - cpustat[CPUTIME_SOFTIRQ ]; softirq_cputime =min (softirq_cputime, maxtime); cpustat[CPUTIME_SOFTIRQ ] += softirq_cputime; -local_irq_restore (flags); +return softirq_cputime; }
Signature · 1 line
2.7.0
Re: [PATCH 2/5] irqtime: Remove needless IRQs disablement on kcpustat update
Quoted 2 lines · Frederic Weisbecker
The callers of the functions performing irqtime kcpustat updates have IRQS disabled, no need to disable them again.
They do, but perhaps this should be annotated through some sparse magic.
It's starting to be hairy, with the requirement spanning many separate
files.
Something like
#define __irq_disabled __must_hold(IRQ)
together with __acquire and __release annotations in include/linux/irqflags.h would do. I'm not sure how to handle local_irq_save/local_irq_restore, but I guess sparse would be fine with
((void)({
raw_local_irq_save(flags);
if (flags) __acquire(IRQ);
}))
and
((void)({
if (flags) __release(IRQ);
raw_local_irq_restore(flags);
}))
since below that it's assembly.
Starting from irqtime_account_hi_update, irqtime_account_si_update and irqtime_account_irq you'd get quite a few functions annotated.
Paolo
Quoted 50 lines
Cc Rik van Riel <[email protected]>Cc Paolo Bonzini <[email protected]>Cc Wanpeng Li <[email protected]>Cc Mike Galbraith <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Ingo Molnar <[email protected]>Cc Eric Dumazet <[email protected]>Signed-off-by Frederic Weisbecker <[email protected]>--- kernel/sched/cputime.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-)kernel/sched/cputime.c +2
-6@@ -78,30 +78,26 @@ EXPORT_SYMBOL_GPL(irqtime_account_irq);static cputime_t irqtime_account_hi_update (cputime_t maxtime) {u64 *cpustat = kcpustat_this_cpu->cpustat; -unsigned long flags;cputime_t irq_cputime; -local_irq_save (flags); irq_cputime =nsecs_to_cputime64 (__this_cpu_read (cpu_hardirq_time)) - cpustat[CPUTIME_IRQ ]; irq_cputime =min (irq_cputime, maxtime); cpustat[CPUTIME_IRQ ] += irq_cputime; -local_irq_restore (flags); +return irq_cputime; }static cputime_t irqtime_account_si_update (cputime_t maxtime) {u64 *cpustat = kcpustat_this_cpu->cpustat; -unsigned long flags;cputime_t softirq_cputime; -local_irq_save (flags); softirq_cputime =nsecs_to_cputime64 (__this_cpu_read (cpu_softirq_time)) - cpustat[CPUTIME_SOFTIRQ ]; softirq_cputime =min (softirq_cputime, maxtime); cpustat[CPUTIME_SOFTIRQ ] += softirq_cputime; -local_irq_restore (flags); +return softirq_cputime; }
Peter Zijlstra
maintainerRe: [PATCH 2/5] irqtime: Remove needless IRQs disablement on kcpustat update
Quoted 9 lines · Paolo Bonzini
Quoted 2 lines · Frederic Weisbecker
The callers of the functions performing irqtime kcpustat updates have IRQS disabled, no need to disable them again.
They do, but perhaps this should be annotated through some sparse magic.
It's starting to be hairy, with the requirement spanning many separate
files.
Sparse sucks for those things...
maybe just add something like lockdep_assert_irqsoff(). Such a call both documents the requirement and validates at runtime when CONFIG_LOCKDEP.
Re: [PATCH 2/5] irqtime: Remove needless IRQs disablement on kcpustat update
In reply to Frederic WeisbeckerQuoted 11 lines · Frederic Weisbecker
The callers of the functions performing irqtime kcpustat updates have IRQS disabled, no need to disable them again.
Cc Rik van Riel <[email protected]>Cc Paolo Bonzini <[email protected]>Cc Wanpeng Li <[email protected]>Cc Mike Galbraith <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Ingo Molnar <[email protected]>Cc Eric Dumazet <[email protected]>Signed-off-by Frederic Weisbecker <[email protected]>
Signature · 1 line
All Rights Reversed.
Peter Zijlstra
maintainerRe: [PATCH 2/5] irqtime: Remove needless IRQs disablement on kcpustat update
In reply to Frederic Weisbeckerkernel/sched/cputime.c EXPORT_SYMBOL_GPL(irqtime_account_irq); · line 82
Quoted 11 lines · Frederic Weisbecker
kernel/sched/cputime.c +0
-1@@ -78,30 +78,26 @@ EXPORT_SYMBOL_GPL(irqtime_account_irq);static cputime_t irqtime_account_hi_update (cputime_t maxtime) {u64 *cpustat = kcpustat_this_cpu->cpustat; -unsigned long flags;cputime_t irq_cputime;
+ lockdep_assert_irqs_disabled();
Quoted 6 lines
kernel/sched/cputime.c +0
-2-local_irq_save (flags); irq_cputime =nsecs_to_cputime64 (__this_cpu_read (cpu_hardirq_time)) - cpustat[CPUTIME_IRQ ]; irq_cputime =min (irq_cputime, maxtime); cpustat[CPUTIME_IRQ ] += irq_cputime; -local_irq_restore (flags);
kernel/sched/cputime.c EXPORT_SYMBOL_GPL(irqtime_account_irq); · line 95
Quoted 10 lines
kernel/sched/cputime.c +1
-1+return irq_cputime; }static cputime_t irqtime_account_si_update (cputime_t maxtime) {u64 *cpustat = kcpustat_this_cpu->cpustat; -unsigned long flags;cputime_t softirq_cputime;
+ lockdep_assert_irqs_disabled();
kernel/sched/cputime.c EXPORT_SYMBOL_GPL(irqtime_account_irq); · line 102
Quoted 9 lines
kernel/sched/cputime.c +1
-2-local_irq_save (flags); softirq_cputime =nsecs_to_cputime64 (__this_cpu_read (cpu_softirq_time)) - cpustat[CPUTIME_SOFTIRQ ]; softirq_cputime =min (softirq_cputime, maxtime); cpustat[CPUTIME_SOFTIRQ ] += softirq_cputime; -local_irq_restore (flags); +return softirq_cputime; }
---
Subject: locking/lockdep: Provide IRQ state assertion helpers
Provide a cheap alternative to:
WARN_ON(!irqs_disabled());
This patch provides:
lockdep_assert_irqs_disabled(); lockdep_assert_softirqs_disabled();
Which compile away for CONFIG_LOCKDEP=n kernels.
---
include/linux/lockdep.h +17 -0
@@ -526,6 +526,23 @@ static inline void print_irqtrace_events }#endif +#if defined(CONFIG_LOCKDEP ) && defined(CONFIG_TRACE_IRQFLAGS ) + +#define lockdep_assert_irqs_disabled () do { \ + WARN_ON(debug_locks && current->hardirqs_enabled); \ + } while (0) + +#define lockdep_assert_softirqs_disabled () do { \ + WARN_ON(debug_locks && current->softirqs_enabled); \ + } while (0) + +#else + +#define lockdep_assert_irqs_disabled () do { } while (0) +#define lockdep_assert_softirqs_disabled () do { } while (0) + +#endif +/* * For trivial one-depth nesting of a lock-class, the following * global define can be used. (Subsystems with multiple levels
Re: [PATCH 2/5] irqtime: Remove needless IRQs disablement on kcpustat update
Quoted 75 lines · Peter Zijlstra
Quoted 11 lines · Frederic Weisbecker
kernel/sched/cputime.c +0
-1@@ -78,30 +78,26 @@ EXPORT_SYMBOL_GPL(irqtime_account_irq);static cputime_t irqtime_account_hi_update (cputime_t maxtime) {u64 *cpustat = kcpustat_this_cpu->cpustat; -unsigned long flags;cputime_t irq_cputime;(quoted diff) +1
-0+ lockdep_assert_irqs_disabled();Quoted 17 lines
kernel/sched/cputime.c +1
-3-local_irq_save (flags); irq_cputime =nsecs_to_cputime64 (__this_cpu_read (cpu_hardirq_time)) - cpustat[CPUTIME_IRQ ]; irq_cputime =min (irq_cputime, maxtime); cpustat[CPUTIME_IRQ ] += irq_cputime; -local_irq_restore (flags); +return irq_cputime; }static cputime_t irqtime_account_si_update (cputime_t maxtime) {u64 *cpustat = kcpustat_this_cpu->cpustat; -unsigned long flags;cputime_t softirq_cputime;(quoted diff) +1
-0+ lockdep_assert_irqs_disabled();kernel/sched/cputime.c EXPORT_SYMBOL_GPL(irqtime_account_irq); · line 102
Quoted 10 lines
kernel/sched/cputime.c +1
-2-local_irq_save (flags); softirq_cputime =nsecs_to_cputime64 (__this_cpu_read (cpu_softirq_time)) - cpustat[CPUTIME_SOFTIRQ ]; softirq_cputime =min (softirq_cputime, maxtime); cpustat[CPUTIME_SOFTIRQ ] += softirq_cputime; -local_irq_restore (flags); +return softirq_cputime; }---
Subject: locking/lockdep: Provide IRQ state assertion helpersProvide a cheap alternative to:
WARN_ON(!irqs_disabled());This patch provides:
lockdep_assert_irqs_disabled(); lockdep_assert_softirqs_disabled();Which compile away for
CONFIG_LOCKDEP=nkernels.Signed-off-by Peter Zijlstra (Intel) <[email protected]>---
include/linux/lockdep.h +10
-0@@ -526,6 +526,23 @@ static inline void print_irqtrace_events }#endif +#if defined(CONFIG_LOCKDEP ) && defined(CONFIG_TRACE_IRQFLAGS ) + +#define lockdep_assert_irqs_disabled () do { \ + WARN_ON(debug_locks && current->hardirqs_enabled); \ + } while (0) + +#define lockdep_assert_softirqs_disabled () do { \ + WARN_ON(debug_locks && current->softirqs_enabled); \ + } while (0) +
include/linux/lockdep.h static inline void print_irqtrace_events · line 548
Quoted 10 lines
include/linux/lockdep.h +7
-0+#else + +#define lockdep_assert_irqs_disabled () do { } while (0) +#define lockdep_assert_softirqs_disabled () do { } while (0) + +#endif +/* * For trivial one-depth nesting of a lock-class, the following * global define can be used. (Subsystems with multiple levels
I love that! I've seen so many cases where I wanted this runtime check without the overhead of it on production kernels.
I can take that patch, now since this is lockdep code and my series is scheduler's code that depend on it, how can we manage the dependency between the two branches?
Perhaps I can add the lockdep patch in the series, there will likely be no wicked conflicts agains potential changes in the lockdep tree.
Another alternative is to use WARN_ON_(!irqs_disabled()) on my series, then on the lockdep branch we can convert all the potential users including the current one.. The lockdep branch would then depend on the others. That way looks better as I can think of several sites to convert.
Thanks.
Peter Zijlstra
maintainerRe: [PATCH 2/5] irqtime: Remove needless IRQs disablement on kcpustat update
Quoted 8 lines · Frederic Weisbecker
I love that! I've seen so many cases where I wanted this runtime check without the overhead of it on production kernels.
I can take that patch, now since this is lockdep code and my series is scheduler's code that depend on it, how can we manage the dependency between the two branches?
Perhaps I can add the lockdep patch in the series, there will likely be no wicked conflicts agains potential changes in the lockdep tree.
Yeah, just take it along with the rest of the patches.
PATCH 3/5 u64_stats: Introduce IRQs disabled helpers
In reply to Frederic WeisbeckerIntroduce light versions of u64_stats helpers for context where either preempt or IRQs are disabled. This way we can make this library usable by scheduler irqtime accounting which currenty implement its ad-hoc version.
--- include/linux/u64_stats_sync.h | 55 ++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 24 deletions(-)
include/linux/u64_stats_sync.h +31 -24
@@ -103,28 +103,41 @@ static inline void u64_stats_update_end_raw(struct u64_stats_sync *syncp)#endif } +static inline unsigned int __u64_stats_fetch_begin (const struct u64_stats_sync *syncp) +{ +#if BITS_PER_LONG ==32 && defined(CONFIG_SMP ) +return read_seqcount_begin (&syncp->seq); +#else +return 0 ; +#endif +} +static inline unsigned int u64_stats_fetch_begin (const struct u64_stats_sync *syncp) { -#if BITS_PER_LONG ==32 && defined(CONFIG_SMP ) -return read_seqcount_begin (&syncp->seq); -#else -#if BITS_PER_LONG ==32 +#if BITS_PER_LONG ==32 && !defined(CONFIG_SMP )preempt_disable (); +#else +return __u64_stats_fetch_begin (syncp);#endif -return 0 ; +} + +static inline bool __u64_stats_fetch_retry (const struct u64_stats_sync *syncp, +unsigned int start) +{ +#if BITS_PER_LONG ==32 && defined(CONFIG_SMP ) +return read_seqcount_retry (&syncp->seq, start); +#else +return false;#endif }static inline bool u64_stats_fetch_retry (const struct u64_stats_sync *syncp,unsigned int start) { -#if BITS_PER_LONG ==32 && defined(CONFIG_SMP ) -return read_seqcount_retry (&syncp->seq, start); -#else -#if BITS_PER_LONG ==32 +#if BITS_PER_LONG ==32 && !defined(CONFIG_SMP )preempt_enable (); -#endif -return false; +#else +return __u64_stats_fetch_retry (syncp, start);#endif }
@@ -136,26 +149,20 @@ static inline bool u64_stats_fetch_retry(const struct u64_stats_sync *syncp,*/ static inline unsigned int u64_stats_fetch_begin_irq (const struct u64_stats_sync *syncp) { -#if BITS_PER_LONG ==32 && defined(CONFIG_SMP ) -return read_seqcount_begin (&syncp->seq); -#else -#if BITS_PER_LONG ==32 +#if BITS_PER_LONG ==32 && !defined(CONFIG_SMP )local_irq_disable (); -#endif -return 0 ; +#else +return __u64_stats_fetch_begin (syncp);#endif }static inline bool u64_stats_fetch_retry_irq (const struct u64_stats_sync *syncp, -unsigned int start) +unsigned int start) { -#if BITS_PER_LONG ==32 && defined(CONFIG_SMP ) -return read_seqcount_retry (&syncp->seq, start); -#else -#if BITS_PER_LONG ==32 +#if BITS_PER_LONG ==32 && !defined(CONFIG_SMP )local_irq_enable (); -#endif -return false; +#else +return __u64_stats_fetch_retry (syncp, start);#endif }
Signature · 1 line
2.7.0
Re: [PATCH 3/5] u64_stats: Introduce IRQs disabled helpers
include/linux/u64_stats_sync.h static inline void u64_stats_update_end_raw(struct u64_stats_sync *syncp) · line 119
Quoted 9 lines · Frederic Weisbecker
static inline unsigned int
u64_stats_fetch_begin(const struct u64_stats_sync *syncp){include/linux/u64_stats_sync.h +2
-4-#if BITS_PER_LONG ==32 && defined(CONFIG_SMP ) -return read_seqcount_begin (&syncp->seq); -#else -#if BITS_PER_LONG ==32 +#if BITS_PER_LONG ==32 && !defined(CONFIG_SMP )preempt_disable (); +#else
This should be #endif, or this side ends without a "return" statement.
Quoted 4 lines
include/linux/u64_stats_sync.h +2
-1+return __u64_stats_fetch_begin (syncp);#endif -return 0 ; +}
...
include/linux/u64_stats_sync.h static inline void u64_stats_update_end_raw(struct u64_stats_sync *syncp) · line 139
Quoted 13 lines
static inline bool u64_stats_fetch_retry(const struct
u64_stats_sync*syncp,include/linux/u64_stats_sync.h +2
-6unsigned int start) { -#if BITS_PER_LONG ==32 && defined(CONFIG_SMP ) -return read_seqcount_retry (&syncp->seq, start); -#else -#if BITS_PER_LONG ==32 +#if BITS_PER_LONG ==32 && !defined(CONFIG_SMP )preempt_enable (); -#endif -return false; +#else
Same here.
Quoted 3 lines
include/linux/u64_stats_sync.h +1
-0+return __u64_stats_fetch_retry (syncp, start);#endif }
...
include/linux/u64_stats_sync.h static inline bool u64_stats_fetch_retry(const struct u64_stats_sync *syncp, · line 154
Quoted 9 lines
include/linux/u64_stats_sync.h +2
-5-return read_seqcount_begin (&syncp->seq); -#else -#if BITS_PER_LONG ==32 +#if BITS_PER_LONG ==32 && !defined(CONFIG_SMP )local_irq_disable (); -#endif -return 0 ; +#else
Same here.
Quoted 4 lines
include/linux/u64_stats_sync.h +1
-0+return __u64_stats_fetch_begin (syncp);#endif
include/linux/u64_stats_sync.h static inline bool u64_stats_fetch_retry(const struct u64_stats_sync *syncp, · line 164
Quoted 10 lines
include/linux/u64_stats_sync.h +2
-6-#if BITS_PER_LONG ==32 && defined(CONFIG_SMP ) -return read_seqcount_retry (&syncp->seq, start); -#else -#if BITS_PER_LONG ==32 +#if BITS_PER_LONG ==32 && !defined(CONFIG_SMP )local_irq_enable (); -#endif -return false; +#else
Same here.
Review
Quoted 2 lines
(quoted diff) +1
-0+ return __u64_stats_fetch_retry(syncp, start); #endif
Thanks,
Paolo
Re: [PATCH 3/5] u64_stats: Introduce IRQs disabled helpers
Quoted 14 lines · Paolo Bonzini
include/linux/u64_stats_sync.h static inline void u64_stats_update_end_raw(struct u64_stats_sync *syncp) · line 119
Quoted 9 lines · Frederic Weisbecker
static inline unsigned int
u64_stats_fetch_begin(const struct u64_stats_sync *syncp){include/linux/u64_stats_sync.h +2
-4-#if BITS_PER_LONG ==32 && defined(CONFIG_SMP ) -return read_seqcount_begin (&syncp->seq); -#else -#if BITS_PER_LONG ==32 +#if BITS_PER_LONG ==32 && !defined(CONFIG_SMP )preempt_disable (); +#else This should be #endif, or this side ends without a "return" statement.
Good catch!
Thanks Paolo!
PATCH 4/5 irqtime: Consolidate accounting synchronization with u64_stats API
In reply to Frederic WeisbeckerThe irqtime accounting currently implement its own ad hoc implementation of u64_stats API. Lets rather consolidate it with the appropriate library.
--- kernel/sched/cputime.c | 31 +++++++++++++--------------- kernel/sched/sched.h | 55 +++++++++++++++----------------------------------- 2 files changed, 30 insertions(+), 56 deletions(-)
kernel/sched/cputime.c +14 -17
@@ -23,10 +23,8 @@* task when irq is in progress while we read rq->clock. That is a worthy * compromise in place of having locks on each irq in account_system_time. */ -DEFINE_PER_CPU (u64, cpu_hardirq_time); -DEFINE_PER_CPU (u64, cpu_softirq_time); +DEFINE_PER_CPU(struct irqtime, cpu_irqtime); -static DEFINE_PER_CPU(u64, irq_start_time);static int sched_clock_irqtime;void enable_sched_clock_irqtime (void )
@@ -39,16 +37,13 @@ void disable_sched_clock_irqtime(void) sched_clock_irqtime =0 ; } -#ifndef CONFIG_64BIT -DEFINE_PER_CPU (seqcount_t, irq_time_seq); -#endif /* CONFIG_64BIT */ -/* * Called before incrementing preempt_count on {soft,}irq_enter * and before decrementing preempt_count on {soft,}irq_exit. */ void irqtime_account_irq (struct task_struct *curr) { +struct irqtime *irqtime =this_cpu_ptr (&cpu_irqtime);s64 delta;int cpu;
@@ -56,10 +51,10 @@ void irqtime_account_irq(struct task_struct *curr)return ; cpu =smp_processor_id (); - delta =sched_clock_cpu (cpu) -__this_cpu_read (irq_start_time); -__this_cpu_add (irq_start_time, delta); + delta =sched_clock_cpu (cpu) - irqtime->irq_start_time; + irqtime->irq_start_time += delta; -irq_time_write_begin (); +u64_stats_update_begin (&irqtime->sync);/* * We do not account for softirq time from ksoftirqd here. * We want to continue accounting softirq time to ksoftirqd thread
@@ -67,11 +62,11 @@ void irqtime_account_irq(struct task_struct *curr)* that do not consume any time, but still wants to run. */ if (hardirq_count ()) -__this_cpu_add (cpu_hardirq_time, delta); + irqtime->hardirq_time += delta;else if (in_serving_softirq () && curr !=this_cpu_ksoftirqd ()) -__this_cpu_add (cpu_softirq_time, delta); + irqtime->softirq_time += delta; -irq_time_write_end (); +u64_stats_update_end (&irqtime->sync); }EXPORT_SYMBOL_GPL (irqtime_account_irq);
@@ -79,9 +74,10 @@ static cputime_t irqtime_account_hi_update(cputime_t maxtime) {u64 *cpustat = kcpustat_this_cpu->cpustat;cputime_t irq_cputime; +u64 nsecs; - irq_cputime =nsecs_to_cputime64 (__this_cpu_read (cpu_hardirq_time)) - - cpustat[CPUTIME_IRQ ]; + nsecs =__this_cpu_read (cpu_irqtime.hardirq_time); + irq_cputime =nsecs_to_cputime64 (nsecs) - cpustat[CPUTIME_IRQ ]; irq_cputime =min (irq_cputime, maxtime); cpustat[CPUTIME_IRQ ] += irq_cputime;
@@ -92,9 +88,10 @@ static cputime_t irqtime_account_si_update(cputime_t maxtime) {u64 *cpustat = kcpustat_this_cpu->cpustat;cputime_t softirq_cputime; +u64 nsecs; - softirq_cputime =nsecs_to_cputime64 (__this_cpu_read (cpu_softirq_time)) - - cpustat[CPUTIME_SOFTIRQ ]; + nsecs =__this_cpu_read (cpu_irqtime.softirq_time); + softirq_cputime =nsecs_to_cputime64 (nsecs) - cpustat[CPUTIME_SOFTIRQ ]; softirq_cputime =min (softirq_cputime, maxtime); cpustat[CPUTIME_SOFTIRQ ] += softirq_cputime;
kernel/sched/sched.h +16 -39
74 lines not on this page · raw message
Signature · 1 line
2.7.0
Re: [PATCH 4/5] irqtime: Consolidate accounting synchronization with u64_stats API
Quoted 5 lines · Frederic Weisbecker
The irqtime accounting currently implement its own ad hoc implementation
ofu64_statsAPI. Lets rather consolidate it with the appropriate library.
Quoted 8 lines
Cc Rik van Riel <[email protected]>Cc Paolo Bonzini <[email protected]>Cc Wanpeng Li <[email protected]>Cc Mike Galbraith <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Ingo Molnar <[email protected]>Cc Eric Dumazet <[email protected]>Signed-off-by Frederic Weisbecker <[email protected]>
Signature · 1 line
All Rights Reversed.
PATCH 5/5 irqtime: Consolidate irqtime flushing code
In reply to Frederic WeisbeckerThe code performing irqtime nsecs stats flushing to kcpustat is roughly the same for hardirq and softirq. So lets consolidate that common code.
--- kernel/sched/cputime.c | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-)
kernel/sched/cputime.c +11 -15
@@ -70,32 +70,28 @@ void irqtime_account_irq(struct task_struct *curr) }EXPORT_SYMBOL_GPL (irqtime_account_irq); -static cputime_t irqtime_account_hi_update (cputime_t maxtime) +static cputime_t irqtime_account_update (u64 irqtime,int idx,cputime_t maxtime) {u64 *cpustat = kcpustat_this_cpu->cpustat;cputime_t irq_cputime; -u64 nsecs; - nsecs =__this_cpu_read (cpu_irqtime.hardirq_time); - irq_cputime =nsecs_to_cputime64 (nsecs) - cpustat[CPUTIME_IRQ ]; + irq_cputime =nsecs_to_cputime64 (irqtime) - cpustat[idx]; irq_cputime =min (irq_cputime, maxtime); - cpustat[CPUTIME_IRQ ] += irq_cputime; + cpustat[idx] += irq_cputime;return irq_cputime; } +static cputime_t irqtime_account_hi_update (cputime_t maxtime) +{ +return irqtime_account_update (__this_cpu_read (cpu_irqtime.hardirq_time), +CPUTIME_IRQ , maxtime); +} +static cputime_t irqtime_account_si_update (cputime_t maxtime) { -u64 *cpustat = kcpustat_this_cpu->cpustat; -cputime_t softirq_cputime; -u64 nsecs; - - nsecs =__this_cpu_read (cpu_irqtime.softirq_time); - softirq_cputime =nsecs_to_cputime64 (nsecs) - cpustat[CPUTIME_SOFTIRQ ]; - softirq_cputime =min (softirq_cputime, maxtime); - cpustat[CPUTIME_SOFTIRQ ] += softirq_cputime; - -return softirq_cputime; +return irqtime_account_update (__this_cpu_read (cpu_irqtime.softirq_time), +CPUTIME_SOFTIRQ , maxtime); }#else /* CONFIG_IRQ_TIME_ACCOUNTING */
Signature · 1 line
2.7.0
Re: [PATCH 5/5] irqtime: Consolidate irqtime flushing code
Quoted 5 lines · Frederic Weisbecker
The code performing irqtime nsecs stats flushing to kcpustat is roughly
the same for hardirq and softirq. So lets consolidate that common code.
Quoted 9 lines
Cc Rik van Riel <[email protected]>Cc Paolo Bonzini <[email protected]>Cc Wanpeng Li <[email protected]>Cc Mike Galbraith <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Ingo Molnar <[email protected]>Cc Eric Dumazet <[email protected]>Signed-off-by Frederic Weisbecker <[email protected]>
Signature · 1 line
All Rights Reversed.