Skip to contentSkip to conversation

sched/cputime: irqtime cleanups v2

PATCH New Frederic Weisbecker 2 participants 11 messages Sep 26, 2016 – Sep 30, 2016 Scheduler

lore
Share & tools
lore
#0 lore raw

cover sched/cputime: irqtime cleanups v2

This second version fixes the ifdeffery issues reported by Paolo Bonzini and includes Rick's reviewed-by tags.

I also intended to include Peterz's patch for lockdep assertion on irqs disabled but unfortunately I get false positives as irqs_disabled() doesn't always imply !current->hardirqs_enabled. This seems to happen during the boot with IRQs interrupting lockdep internals while current->lockdep_recursion == 1. Therefore the trace_hardirqs_off() on x86 IRQ entry gets ignored. I'll investigate further and try to get that fixed in order to introduce lockdep_assert_irqs_disabled() properly but that's outside the scope of this patchset now.

git://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git
sched/irqtime-v2

HEAD: e336b2491bf02e95ff3387e7d7a8276b22c8b5f3

Thanks,
Frederic
---

Frederic Weisbecker (5):
irqtime: No need for preempt-safe accessors
irqtime: Remove needless IRQs disablement on kcpustat update
u64_stats: Introduce IRQs disabled helpers
irqtime: Consolidate accounting synchronization with u64_stats API
irqtime: Consolidate irqtime flushing code

 include/linux/u64_stats_sync.h | 45 +++++++++++++++++++----------------
 kernel/sched/cputime.c         | 54 ++++++++++++++++++------------------------
 kernel/sched/sched.h           | 53 ++++++++++++-----------------------------
 3 files changed, 62 insertions(+), 90 deletions(-)
#1 lore raw

PATCH 1/5 irqtime: No need for preempt-safe accessors

We can safely use the preempt-unsafe accessors for irqtime when we flush its counters to kcpustat as IRQs are disabled at this time.

Cc Paolo Bonzini <[email protected]>
Cc Wanpeng Li <[email protected]>
Cc Mike Galbraith <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Eric Dumazet <[email protected]>
Cc Rik van Riel <[email protected]>
---
 kernel/sched/cputime.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
kernel/sched/cputime.c +2 -2
index b93c72d..f111076 100644
@@ -82,7 +82,7 @@ static cputime_t irqtime_account_hi_update(cputime_t maxtime)
 	cputime_t irq_cputime;
 
 	local_irq_save(flags);
-	irq_cputime = nsecs_to_cputime64(this_cpu_read(cpu_hardirq_time)) -
+	irq_cputime = nsecs_to_cputime64(__this_cpu_read(cpu_hardirq_time)) -
 		      cpustat[CPUTIME_IRQ];
 	irq_cputime = min(irq_cputime, maxtime);
 	cpustat[CPUTIME_IRQ] += irq_cputime;
@@ -97,7 +97,7 @@ static cputime_t irqtime_account_si_update(cputime_t maxtime)
 	cputime_t softirq_cputime;
 
 	local_irq_save(flags);
-	softirq_cputime = nsecs_to_cputime64(this_cpu_read(cpu_softirq_time)) -
+	softirq_cputime = nsecs_to_cputime64(__this_cpu_read(cpu_softirq_time)) -
 			  cpustat[CPUTIME_SOFTIRQ];
 	softirq_cputime = min(softirq_cputime, maxtime);
 	cpustat[CPUTIME_SOFTIRQ] += softirq_cputime;
Signature · 1 line
2.7.4
#2 lore raw

[tip:sched/core] sched/irqtime: No need for preempt-safe accessors

sched/irqtime: No need for preempt-safe accessors
Commit-ID:  f9094a65755df86ec931f47b781f68ea3095cb56
Gitweb:     http://git.kernel.org/tip/f9094a65755df86ec931f47b781f68ea3095cb56
Author:     Frederic Weisbecker <[email protected]>
AuthorDate: Mon, 26 Sep 2016 02:29:17 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Fri, 30 Sep 2016 11:46:38 +0200

sched/irqtime: No need for preempt-safe accessors

We can safely use the preempt-unsafe accessors for irqtime when we flush its counters to kcpustat as IRQs are disabled at this time.

---
 kernel/sched/cputime.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
kernel/sched/cputime.c +2 -2
index b93c72d..f111076 100644
@@ -82,7 +82,7 @@ static cputime_t irqtime_account_hi_update(cputime_t maxtime)
 	cputime_t irq_cputime;
 
 	local_irq_save(flags);
-	irq_cputime = nsecs_to_cputime64(this_cpu_read(cpu_hardirq_time)) -
+	irq_cputime = nsecs_to_cputime64(__this_cpu_read(cpu_hardirq_time)) -
 		      cpustat[CPUTIME_IRQ];
 	irq_cputime = min(irq_cputime, maxtime);
 	cpustat[CPUTIME_IRQ] += irq_cputime;
@@ -97,7 +97,7 @@ static cputime_t irqtime_account_si_update(cputime_t maxtime)
 	cputime_t softirq_cputime;
 
 	local_irq_save(flags);
-	softirq_cputime = nsecs_to_cputime64(this_cpu_read(cpu_softirq_time)) -
+	softirq_cputime = nsecs_to_cputime64(__this_cpu_read(cpu_softirq_time)) -
 			  cpustat[CPUTIME_SOFTIRQ];
 	softirq_cputime = min(softirq_cputime, maxtime);
 	cpustat[CPUTIME_SOFTIRQ] += softirq_cputime;
#3 lore raw

PATCH 2/5 irqtime: Remove needless IRQs disablement on kcpustat update

In reply to Frederic Weisbecker

The callers of the functions performing irqtime kcpustat updates have IRQS disabled, no need to disable them again.

Cc Paolo Bonzini <[email protected]>
Cc Wanpeng Li <[email protected]>
Cc Mike Galbraith <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Eric Dumazet <[email protected]>
Cc Rik van Riel <[email protected]>
---
 kernel/sched/cputime.c | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)
kernel/sched/cputime.c +5 -6
index f111076..94b1a72 100644
@@ -78,30 +78,26 @@ EXPORT_SYMBOL_GPL(irqtime_account_irq);
 static cputime_t irqtime_account_hi_update(cputime_t maxtime)
 {
 	u64 *cpustat = kcpustat_this_cpu->cpustat;
-	unsigned long flags;
 	cputime_t irq_cputime;
 
-	local_irq_save(flags);
 	irq_cputime = nsecs_to_cputime64(__this_cpu_read(cpu_hardirq_time)) -
 		      cpustat[CPUTIME_IRQ];
 	irq_cputime = min(irq_cputime, maxtime);
 	cpustat[CPUTIME_IRQ] += irq_cputime;
-	local_irq_restore(flags);
+
 	return irq_cputime;
 }
 
 static cputime_t irqtime_account_si_update(cputime_t maxtime)
 {
 	u64 *cpustat = kcpustat_this_cpu->cpustat;
-	unsigned long flags;
 	cputime_t softirq_cputime;
 
-	local_irq_save(flags);
 	softirq_cputime = nsecs_to_cputime64(__this_cpu_read(cpu_softirq_time)) -
 			  cpustat[CPUTIME_SOFTIRQ];
 	softirq_cputime = min(softirq_cputime, maxtime);
 	cpustat[CPUTIME_SOFTIRQ] += softirq_cputime;
-	local_irq_restore(flags);
+
 	return softirq_cputime;
 }
 
@@ -295,6 +291,9 @@ static inline cputime_t account_other_time(cputime_t max)
 {
 	cputime_t accounted;
 
+	/* Shall be converted to a lockdep-enabled lightweight check */
+	WARN_ON_ONCE(!irqs_disabled());
+
 	accounted = steal_account_process_time(max);
 
 	if (accounted < max)
Signature · 1 line
2.7.4
#4 lore raw

[tip:sched/core] sched/irqtime: Remove needless IRQs disablement on kcpustat update

sched/irqtime: Remove needless IRQs disablement on kcpustat update
Commit-ID:  2810f611f908112ea1b30bc016d25205acb3d486
Gitweb:     http://git.kernel.org/tip/2810f611f908112ea1b30bc016d25205acb3d486
Author:     Frederic Weisbecker <[email protected]>
AuthorDate: Mon, 26 Sep 2016 02:29:18 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Fri, 30 Sep 2016 11:46:39 +0200

sched/irqtime: Remove needless IRQs disablement on kcpustat update

The callers of the functions performing irqtime kcpustat updates have IRQS disabled, no need to disable them again.

---
 kernel/sched/cputime.c | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)
kernel/sched/cputime.c +5 -6
index f111076..94b1a72 100644
@@ -78,30 +78,26 @@ EXPORT_SYMBOL_GPL(irqtime_account_irq);
 static cputime_t irqtime_account_hi_update(cputime_t maxtime)
 {
 	u64 *cpustat = kcpustat_this_cpu->cpustat;
-	unsigned long flags;
 	cputime_t irq_cputime;
 
-	local_irq_save(flags);
 	irq_cputime = nsecs_to_cputime64(__this_cpu_read(cpu_hardirq_time)) -
 		      cpustat[CPUTIME_IRQ];
 	irq_cputime = min(irq_cputime, maxtime);
 	cpustat[CPUTIME_IRQ] += irq_cputime;
-	local_irq_restore(flags);
+
 	return irq_cputime;
 }
 
 static cputime_t irqtime_account_si_update(cputime_t maxtime)
 {
 	u64 *cpustat = kcpustat_this_cpu->cpustat;
-	unsigned long flags;
 	cputime_t softirq_cputime;
 
-	local_irq_save(flags);
 	softirq_cputime = nsecs_to_cputime64(__this_cpu_read(cpu_softirq_time)) -
 			  cpustat[CPUTIME_SOFTIRQ];
 	softirq_cputime = min(softirq_cputime, maxtime);
 	cpustat[CPUTIME_SOFTIRQ] += softirq_cputime;
-	local_irq_restore(flags);
+
 	return softirq_cputime;
 }
 
@@ -295,6 +291,9 @@ static inline cputime_t account_other_time(cputime_t max)
 {
 	cputime_t accounted;
 
+	/* Shall be converted to a lockdep-enabled lightweight check */
+	WARN_ON_ONCE(!irqs_disabled());
+
 	accounted = steal_account_process_time(max);
 
 	if (accounted < max)
#5 lore raw

PATCH 3/5 u64_stats: Introduce IRQs disabled helpers

In reply to Frederic Weisbecker

Introduce light versions of u64_stats helpers for context where either preempt or IRQs are disabled. This way we can make this library usable by scheduler irqtime accounting which currenty implement its ad-hoc version.

Cc Rik van Riel <[email protected]>
Cc Paolo Bonzini <[email protected]>
Cc Wanpeng Li <[email protected]>
Cc Mike Galbraith <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Eric Dumazet <[email protected]>
---
 include/linux/u64_stats_sync.h | 51 ++++++++++++++++++++++--------------------
 1 file changed, 27 insertions(+), 24 deletions(-)
include/linux/u64_stats_sync.h +27 -24
index d3a2bb7..650f3dd 100644
@@ -103,29 +103,40 @@ static inline void u64_stats_update_end_raw(struct u64_stats_sync *syncp)
 #endif
 }
 
+static inline unsigned int __u64_stats_fetch_begin(const struct u64_stats_sync *syncp)
+{
+#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
+	return read_seqcount_begin(&syncp->seq);
+#else
+	return 0;
+#endif
+}
+
 static inline unsigned int u64_stats_fetch_begin(const struct u64_stats_sync *syncp)
 {
-#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
-	return read_seqcount_begin(&syncp->seq);
-#else
-#if BITS_PER_LONG==32
+#if BITS_PER_LONG==32 && !defined(CONFIG_SMP)
 	preempt_disable();
 #endif
-	return 0;
+	return __u64_stats_fetch_begin(syncp);
+}
+
+static inline bool __u64_stats_fetch_retry(const struct u64_stats_sync *syncp,
+					 unsigned int start)
+{
+#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
+	return read_seqcount_retry(&syncp->seq, start);
+#else
+	return false;
 #endif
 }
 
 static inline bool u64_stats_fetch_retry(const struct u64_stats_sync *syncp,
 					 unsigned int start)
 {
-#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
-	return read_seqcount_retry(&syncp->seq, start);
-#else
-#if BITS_PER_LONG==32
+#if BITS_PER_LONG==32 && !defined(CONFIG_SMP)
 	preempt_enable();
 #endif
-	return false;
-#endif
+	return __u64_stats_fetch_retry(syncp, start);
 }
 
 /*
@@ -136,27 +147,19 @@ static inline bool u64_stats_fetch_retry(const struct u64_stats_sync *syncp,
  */
 static inline unsigned int u64_stats_fetch_begin_irq(const struct u64_stats_sync *syncp)
 {
-#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
-	return read_seqcount_begin(&syncp->seq);
-#else
-#if BITS_PER_LONG==32
+#if BITS_PER_LONG==32 && !defined(CONFIG_SMP)
 	local_irq_disable();
 #endif
-	return 0;
-#endif
+	return __u64_stats_fetch_begin(syncp);
 }
 
 static inline bool u64_stats_fetch_retry_irq(const struct u64_stats_sync *syncp,
-					 unsigned int start)
+					     unsigned int start)
 {
-#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
-	return read_seqcount_retry(&syncp->seq, start);
-#else
-#if BITS_PER_LONG==32
+#if BITS_PER_LONG==32 && !defined(CONFIG_SMP)
 	local_irq_enable();
 #endif
-	return false;
-#endif
+	return __u64_stats_fetch_retry(syncp, start);
 }
 
 #endif /* _LINUX_U64_STATS_SYNC_H */
Signature · 1 line
2.7.4
#6 lore raw

[tip:sched/core] u64_stats: Introduce IRQs disabled helpers

u64_stats: Introduce IRQs disabled helpers
Commit-ID:  68107df5f2cb5dc3785be40162bfe2f19a178bbb
Gitweb:     http://git.kernel.org/tip/68107df5f2cb5dc3785be40162bfe2f19a178bbb
Author:     Frederic Weisbecker <[email protected]>
AuthorDate: Mon, 26 Sep 2016 02:29:19 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Fri, 30 Sep 2016 11:46:40 +0200

u64_stats: Introduce IRQs disabled helpers

Introduce light versions of u64_stats helpers for context where either preempt or IRQs are disabled. This way we can make this library usable by scheduler irqtime accounting which currenty implement its ad-hoc version.

Cc Eric Dumazet <[email protected]>
Cc Linus Torvalds <[email protected]>
Cc Mike Galbraith <[email protected]>
Cc Paolo Bonzini <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Rik van Riel <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Wanpeng Li <[email protected]>
---
 include/linux/u64_stats_sync.h | 45 ++++++++++++++++++++++--------------------
 1 file changed, 24 insertions(+), 21 deletions(-)
include/linux/u64_stats_sync.h +24 -21
index d3a2bb7..650f3dd 100644
@@ -103,31 +103,42 @@ static inline void u64_stats_update_end_raw(struct u64_stats_sync *syncp)
 #endif
 }
 
-static inline unsigned int u64_stats_fetch_begin(const struct u64_stats_sync *syncp)
+static inline unsigned int __u64_stats_fetch_begin(const struct u64_stats_sync *syncp)
 {
 #if BITS_PER_LONG==32 && defined(CONFIG_SMP)
 	return read_seqcount_begin(&syncp->seq);
 #else
-#if BITS_PER_LONG==32
-	preempt_disable();
-#endif
 	return 0;
 #endif
 }
 
-static inline bool u64_stats_fetch_retry(const struct u64_stats_sync *syncp,
+static inline unsigned int u64_stats_fetch_begin(const struct u64_stats_sync *syncp)
+{
+#if BITS_PER_LONG==32 && !defined(CONFIG_SMP)
+	preempt_disable();
+#endif
+	return __u64_stats_fetch_begin(syncp);
+}
+
+static inline bool __u64_stats_fetch_retry(const struct u64_stats_sync *syncp,
 					 unsigned int start)
 {
 #if BITS_PER_LONG==32 && defined(CONFIG_SMP)
 	return read_seqcount_retry(&syncp->seq, start);
 #else
-#if BITS_PER_LONG==32
-	preempt_enable();
-#endif
 	return false;
 #endif
 }
 
+static inline bool u64_stats_fetch_retry(const struct u64_stats_sync *syncp,
+					 unsigned int start)
+{
+#if BITS_PER_LONG==32 && !defined(CONFIG_SMP)
+	preempt_enable();
+#endif
+	return __u64_stats_fetch_retry(syncp, start);
+}
+
 /*
  * In case irq handlers can update u64 counters, readers can use following helpers
  * - SMP 32bit arches use seqcount protection, irq safe.
@@ -136,27 +147,19 @@ static inline bool u64_stats_fetch_retry(const struct u64_stats_sync *syncp,
  */
 static inline unsigned int u64_stats_fetch_begin_irq(const struct u64_stats_sync *syncp)
 {
-#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
-	return read_seqcount_begin(&syncp->seq);
-#else
-#if BITS_PER_LONG==32
+#if BITS_PER_LONG==32 && !defined(CONFIG_SMP)
 	local_irq_disable();
 #endif
-	return 0;
-#endif
+	return __u64_stats_fetch_begin(syncp);
 }
 
 static inline bool u64_stats_fetch_retry_irq(const struct u64_stats_sync *syncp,
-					 unsigned int start)
+					     unsigned int start)
 {
-#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
-	return read_seqcount_retry(&syncp->seq, start);
-#else
-#if BITS_PER_LONG==32
+#if BITS_PER_LONG==32 && !defined(CONFIG_SMP)
 	local_irq_enable();
 #endif
-	return false;
-#endif
+	return __u64_stats_fetch_retry(syncp, start);
 }
 
 #endif /* _LINUX_U64_STATS_SYNC_H */
#7 lore raw

PATCH 4/5 irqtime: Consolidate accounting synchronization with u64_stats API

In reply to Frederic Weisbecker

The irqtime accounting currently implement its own ad hoc implementation of u64_stats API. Lets rather consolidate it with the appropriate library.

Cc Paolo Bonzini <[email protected]>
Cc Wanpeng Li <[email protected]>
Cc Mike Galbraith <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Eric Dumazet <[email protected]>
Cc Rik van Riel <[email protected]>
---
 kernel/sched/cputime.c | 31 +++++++++++++---------------
 kernel/sched/sched.h   | 55 +++++++++++++++-----------------------------------
 2 files changed, 30 insertions(+), 56 deletions(-)
kernel/sched/cputime.c +14 -17
index 94b1a72..1cea2f1 100644
@@ -23,10 +23,8 @@
  * task when irq is in progress while we read rq->clock. That is a worthy
  * compromise in place of having locks on each irq in account_system_time.
  */
-DEFINE_PER_CPU(u64, cpu_hardirq_time);
-DEFINE_PER_CPU(u64, cpu_softirq_time);
+DEFINE_PER_CPU(struct irqtime, cpu_irqtime);
 
-static DEFINE_PER_CPU(u64, irq_start_time);
 static int sched_clock_irqtime;
 
 void enable_sched_clock_irqtime(void)
@@ -39,16 +37,13 @@ void disable_sched_clock_irqtime(void)
 	sched_clock_irqtime = 0;
 }
 
-#ifndef CONFIG_64BIT
-DEFINE_PER_CPU(seqcount_t, irq_time_seq);
-#endif /* CONFIG_64BIT */
-
 /*
  * Called before incrementing preempt_count on {soft,}irq_enter
  * and before decrementing preempt_count on {soft,}irq_exit.
  */
 void irqtime_account_irq(struct task_struct *curr)
 {
+	struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime);
 	s64 delta;
 	int cpu;
 
@@ -56,10 +51,10 @@ void irqtime_account_irq(struct task_struct *curr)
 		return;
 
 	cpu = smp_processor_id();
-	delta = sched_clock_cpu(cpu) - __this_cpu_read(irq_start_time);
-	__this_cpu_add(irq_start_time, delta);
+	delta = sched_clock_cpu(cpu) - irqtime->irq_start_time;
+	irqtime->irq_start_time += delta;
 
-	irq_time_write_begin();
+	u64_stats_update_begin(&irqtime->sync);
 	/*
 	 * We do not account for softirq time from ksoftirqd here.
 	 * We want to continue accounting softirq time to ksoftirqd thread
@@ -67,11 +62,11 @@ void irqtime_account_irq(struct task_struct *curr)
 	 * that do not consume any time, but still wants to run.
 	 */
 	if (hardirq_count())
-		__this_cpu_add(cpu_hardirq_time, delta);
+		irqtime->hardirq_time += delta;
 	else if (in_serving_softirq() && curr != this_cpu_ksoftirqd())
-		__this_cpu_add(cpu_softirq_time, delta);
+		irqtime->softirq_time += delta;
 
-	irq_time_write_end();
+	u64_stats_update_end(&irqtime->sync);
 }
 EXPORT_SYMBOL_GPL(irqtime_account_irq);
 
@@ -79,9 +74,10 @@ static cputime_t irqtime_account_hi_update(cputime_t maxtime)
 {
 	u64 *cpustat = kcpustat_this_cpu->cpustat;
 	cputime_t irq_cputime;
+	u64 nsecs;
 
-	irq_cputime = nsecs_to_cputime64(__this_cpu_read(cpu_hardirq_time)) -
-		      cpustat[CPUTIME_IRQ];
+	nsecs = __this_cpu_read(cpu_irqtime.hardirq_time);
+	irq_cputime = nsecs_to_cputime64(nsecs) - cpustat[CPUTIME_IRQ];
 	irq_cputime = min(irq_cputime, maxtime);
 	cpustat[CPUTIME_IRQ] += irq_cputime;
 
@@ -92,9 +88,10 @@ static cputime_t irqtime_account_si_update(cputime_t maxtime)
 {
 	u64 *cpustat = kcpustat_this_cpu->cpustat;
 	cputime_t softirq_cputime;
+	u64 nsecs;
 
-	softirq_cputime = nsecs_to_cputime64(__this_cpu_read(cpu_softirq_time)) -
-			  cpustat[CPUTIME_SOFTIRQ];
+	nsecs = __this_cpu_read(cpu_irqtime.softirq_time);
+	softirq_cputime = nsecs_to_cputime64(nsecs) - cpustat[CPUTIME_SOFTIRQ];
 	softirq_cputime = min(softirq_cputime, maxtime);
 	cpustat[CPUTIME_SOFTIRQ] += softirq_cputime;
 
kernel/sched/sched.h +16 -39

74 lines not on this page · raw message

Signature · 1 line
2.7.4
#8 lore raw

[tip:sched/core] sched/irqtime: Consolidate accounting synchronization with u64_stats API

sched/irqtime: Consolidate accounting synchronization with u64_stats API
Commit-ID:  19d23dbfeb10724675152915e76e03d771f23d9d
Gitweb:     http://git.kernel.org/tip/19d23dbfeb10724675152915e76e03d771f23d9d
Author:     Frederic Weisbecker <[email protected]>
AuthorDate: Mon, 26 Sep 2016 02:29:20 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Fri, 30 Sep 2016 11:46:40 +0200

sched/irqtime: Consolidate accounting synchronization with u64_stats API

The irqtime accounting currently implement its own ad hoc implementation of u64_stats API. Lets rather consolidate it with the appropriate library.

---
 kernel/sched/cputime.c | 31 +++++++++++++----------------
 kernel/sched/sched.h   | 53 ++++++++++++++------------------------------------
 2 files changed, 29 insertions(+), 55 deletions(-)
kernel/sched/cputime.c +14 -17
index 94b1a72..1cea2f1 100644
@@ -23,10 +23,8 @@
  * task when irq is in progress while we read rq->clock. That is a worthy
  * compromise in place of having locks on each irq in account_system_time.
  */
-DEFINE_PER_CPU(u64, cpu_hardirq_time);
-DEFINE_PER_CPU(u64, cpu_softirq_time);
+DEFINE_PER_CPU(struct irqtime, cpu_irqtime);
 
-static DEFINE_PER_CPU(u64, irq_start_time);
 static int sched_clock_irqtime;
 
 void enable_sched_clock_irqtime(void)
@@ -39,16 +37,13 @@ void disable_sched_clock_irqtime(void)
 	sched_clock_irqtime = 0;
 }
 
-#ifndef CONFIG_64BIT
-DEFINE_PER_CPU(seqcount_t, irq_time_seq);
-#endif /* CONFIG_64BIT */
-
 /*
  * Called before incrementing preempt_count on {soft,}irq_enter
  * and before decrementing preempt_count on {soft,}irq_exit.
  */
 void irqtime_account_irq(struct task_struct *curr)
 {
+	struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime);
 	s64 delta;
 	int cpu;
 
@@ -56,10 +51,10 @@ void irqtime_account_irq(struct task_struct *curr)
 		return;
 
 	cpu = smp_processor_id();
-	delta = sched_clock_cpu(cpu) - __this_cpu_read(irq_start_time);
-	__this_cpu_add(irq_start_time, delta);
+	delta = sched_clock_cpu(cpu) - irqtime->irq_start_time;
+	irqtime->irq_start_time += delta;
 
-	irq_time_write_begin();
+	u64_stats_update_begin(&irqtime->sync);
 	/*
 	 * We do not account for softirq time from ksoftirqd here.
 	 * We want to continue accounting softirq time to ksoftirqd thread
@@ -67,11 +62,11 @@ void irqtime_account_irq(struct task_struct *curr)
 	 * that do not consume any time, but still wants to run.
 	 */
 	if (hardirq_count())
-		__this_cpu_add(cpu_hardirq_time, delta);
+		irqtime->hardirq_time += delta;
 	else if (in_serving_softirq() && curr != this_cpu_ksoftirqd())
-		__this_cpu_add(cpu_softirq_time, delta);
+		irqtime->softirq_time += delta;
 
-	irq_time_write_end();
+	u64_stats_update_end(&irqtime->sync);
 }
 EXPORT_SYMBOL_GPL(irqtime_account_irq);
 
@@ -79,9 +74,10 @@ static cputime_t irqtime_account_hi_update(cputime_t maxtime)
 {
 	u64 *cpustat = kcpustat_this_cpu->cpustat;
 	cputime_t irq_cputime;
+	u64 nsecs;
 
-	irq_cputime = nsecs_to_cputime64(__this_cpu_read(cpu_hardirq_time)) -
-		      cpustat[CPUTIME_IRQ];
+	nsecs = __this_cpu_read(cpu_irqtime.hardirq_time);
+	irq_cputime = nsecs_to_cputime64(nsecs) - cpustat[CPUTIME_IRQ];
 	irq_cputime = min(irq_cputime, maxtime);
 	cpustat[CPUTIME_IRQ] += irq_cputime;
 
@@ -92,9 +88,10 @@ static cputime_t irqtime_account_si_update(cputime_t maxtime)
 {
 	u64 *cpustat = kcpustat_this_cpu->cpustat;
 	cputime_t softirq_cputime;
+	u64 nsecs;
 
-	softirq_cputime = nsecs_to_cputime64(__this_cpu_read(cpu_softirq_time)) -
-			  cpustat[CPUTIME_SOFTIRQ];
+	nsecs = __this_cpu_read(cpu_irqtime.softirq_time);
+	softirq_cputime = nsecs_to_cputime64(nsecs) - cpustat[CPUTIME_SOFTIRQ];
 	softirq_cputime = min(softirq_cputime, maxtime);
 	cpustat[CPUTIME_SOFTIRQ] += softirq_cputime;
 
kernel/sched/sched.h +15 -38

73 lines not on this page · raw message

#9 lore raw

PATCH 5/5 irqtime: Consolidate irqtime flushing code

In reply to Frederic Weisbecker

The code performing irqtime nsecs stats flushing to kcpustat is roughly the same for hardirq and softirq. So lets consolidate that common code.

Cc Paolo Bonzini <[email protected]>
Cc Wanpeng Li <[email protected]>
Cc Mike Galbraith <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Eric Dumazet <[email protected]>
Cc Rik van Riel <[email protected]>
---
 kernel/sched/cputime.c | 26 +++++++++++---------------
 1 file changed, 11 insertions(+), 15 deletions(-)
kernel/sched/cputime.c +11 -15
index 1cea2f1..5ebee31 100644
@@ -70,32 +70,28 @@ void irqtime_account_irq(struct task_struct *curr)
 }
 EXPORT_SYMBOL_GPL(irqtime_account_irq);
 
-static cputime_t irqtime_account_hi_update(cputime_t maxtime)
+static cputime_t irqtime_account_update(u64 irqtime, int idx, cputime_t maxtime)
 {
 	u64 *cpustat = kcpustat_this_cpu->cpustat;
 	cputime_t irq_cputime;
-	u64 nsecs;
 
-	nsecs = __this_cpu_read(cpu_irqtime.hardirq_time);
-	irq_cputime = nsecs_to_cputime64(nsecs) - cpustat[CPUTIME_IRQ];
+	irq_cputime = nsecs_to_cputime64(irqtime) - cpustat[idx];
 	irq_cputime = min(irq_cputime, maxtime);
-	cpustat[CPUTIME_IRQ] += irq_cputime;
+	cpustat[idx] += irq_cputime;
 
 	return irq_cputime;
 }
 
+static cputime_t irqtime_account_hi_update(cputime_t maxtime)
+{
+	return irqtime_account_update(__this_cpu_read(cpu_irqtime.hardirq_time),
+				      CPUTIME_IRQ, maxtime);
+}
+
 static cputime_t irqtime_account_si_update(cputime_t maxtime)
 {
-	u64 *cpustat = kcpustat_this_cpu->cpustat;
-	cputime_t softirq_cputime;
-	u64 nsecs;
-
-	nsecs = __this_cpu_read(cpu_irqtime.softirq_time);
-	softirq_cputime = nsecs_to_cputime64(nsecs) - cpustat[CPUTIME_SOFTIRQ];
-	softirq_cputime = min(softirq_cputime, maxtime);
-	cpustat[CPUTIME_SOFTIRQ] += softirq_cputime;
-
-	return softirq_cputime;
+	return irqtime_account_update(__this_cpu_read(cpu_irqtime.softirq_time),
+				      CPUTIME_SOFTIRQ, maxtime);
 }
 
 #else /* CONFIG_IRQ_TIME_ACCOUNTING */
Signature · 1 line
2.7.4
#10 lore raw

[tip:sched/core] sched/irqtime: Consolidate irqtime flushing code

sched/irqtime: Consolidate irqtime flushing code
Commit-ID:  447976ef4fd09b1be88b316d1a81553f1aa7cd07
Gitweb:     http://git.kernel.org/tip/447976ef4fd09b1be88b316d1a81553f1aa7cd07
Author:     Frederic Weisbecker <[email protected]>
AuthorDate: Mon, 26 Sep 2016 02:29:21 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Fri, 30 Sep 2016 11:46:41 +0200

sched/irqtime: Consolidate irqtime flushing code

The code performing irqtime nsecs stats flushing to kcpustat is roughly the same for hardirq and softirq. So lets consolidate that common code.

---
 kernel/sched/cputime.c | 26 +++++++++++---------------
 1 file changed, 11 insertions(+), 15 deletions(-)
kernel/sched/cputime.c +11 -15
index 1cea2f1..5ebee31 100644
@@ -70,32 +70,28 @@ void irqtime_account_irq(struct task_struct *curr)
 }
 EXPORT_SYMBOL_GPL(irqtime_account_irq);
 
-static cputime_t irqtime_account_hi_update(cputime_t maxtime)
+static cputime_t irqtime_account_update(u64 irqtime, int idx, cputime_t maxtime)
 {
 	u64 *cpustat = kcpustat_this_cpu->cpustat;
 	cputime_t irq_cputime;
-	u64 nsecs;
 
-	nsecs = __this_cpu_read(cpu_irqtime.hardirq_time);
-	irq_cputime = nsecs_to_cputime64(nsecs) - cpustat[CPUTIME_IRQ];
+	irq_cputime = nsecs_to_cputime64(irqtime) - cpustat[idx];
 	irq_cputime = min(irq_cputime, maxtime);
-	cpustat[CPUTIME_IRQ] += irq_cputime;
+	cpustat[idx] += irq_cputime;
 
 	return irq_cputime;
 }
 
-static cputime_t irqtime_account_si_update(cputime_t maxtime)
+static cputime_t irqtime_account_hi_update(cputime_t maxtime)
 {
-	u64 *cpustat = kcpustat_this_cpu->cpustat;
-	cputime_t softirq_cputime;
-	u64 nsecs;
-
-	nsecs = __this_cpu_read(cpu_irqtime.softirq_time);
-	softirq_cputime = nsecs_to_cputime64(nsecs) - cpustat[CPUTIME_SOFTIRQ];
-	softirq_cputime = min(softirq_cputime, maxtime);
-	cpustat[CPUTIME_SOFTIRQ] += softirq_cputime;
+	return irqtime_account_update(__this_cpu_read(cpu_irqtime.hardirq_time),
+				      CPUTIME_IRQ, maxtime);
+}
 
-	return softirq_cputime;
+static cputime_t irqtime_account_si_update(cputime_t maxtime)
+{
+	return irqtime_account_update(__this_cpu_read(cpu_irqtime.softirq_time),
+				      CPUTIME_SOFTIRQ, maxtime);
 }
 
 #else /* CONFIG_IRQ_TIME_ACCOUNTING */