Skip to contentSkip to conversation

sched: update load count only once per cpu in 10 tick update window

PATCH In review Chase Douglas 3 participants 10 messages Apr 13, 2010 – Apr 23, 2010

lore
Share & tools
lore
#0 lore raw

PATCH v3 1/1 sched: update load count only once per cpu in 10 tick update window

There's a period of 10 ticks where calc_load_tasks is updated by all the cpus for the load avg. Usually all the cpus do this during the first tick. If any cpus go idle, calc_load_tasks is decremented accordingly. However, if they wake up calc_load_tasks is not incremented. Thus, if cpus go idle during the 10 tick period, calc_load_tasks may be decremented to a non-representative value. This issue can lead to systems having a load avg of exactly 0, even though the real load avg could theoretically be up to NR_CPUS.

This change defers calc_load_tasks accounting after each cpu updates the count until after the 10 tick update window.

A few points:

* A global atomic deferral counter, and not per-cpu vars, is needed because a cpu may go NOHZ idle and not be able to update the global calc_load_tasks variable for subsequent load calculations.
* It is not enough to add calls to account for the load when a cpu is awakened:
- Load avg calculation must be independent of cpu load.
- If a cpu is awakend by one tasks, but then has more scheduled before the end of the update window, only the first task will be accounted.

---
 kernel/sched.c |   24 ++++++++++++++++++++++--
 1 files changed, 22 insertions(+), 2 deletions(-)
kernel/sched.c +22 -2
index abb36b1..be348cd 100644
@@ -3010,6 +3010,7 @@ unsigned long this_cpu_load(void)
 
 /* Variables and functions for calc_load */
 static atomic_long_t calc_load_tasks;
+static atomic_long_t calc_load_tasks_deferred;
 static unsigned long calc_load_update;
 unsigned long avenrun[3];
 EXPORT_SYMBOL(avenrun);
@@ -3064,7 +3065,7 @@ void calc_global_load(void)
  */
 static void calc_load_account_active(struct rq *this_rq)
 {
-	long nr_active, delta;
+	long nr_active, delta, deferred;
 
 	nr_active = this_rq->nr_running;
 	nr_active += (long) this_rq->nr_uninterruptible;
@@ -3072,6 +3073,25 @@ static void calc_load_account_active(struct rq *this_rq)
 	if (nr_active != this_rq->calc_load_active) {
 		delta = nr_active - this_rq->calc_load_active;
 		this_rq->calc_load_active = nr_active;
+
+		/*
+		 * Update calc_load_tasks only once per cpu in 10 tick update
+		 * window.
+		 */
+		if (unlikely(time_before(jiffies, this_rq->calc_load_update) &&
+			     time_after_eq(jiffies, calc_load_update))) {
+			if (delta)
+				atomic_long_add(delta,
+						&calc_load_tasks_deferred);
+			return;
+		}
+
+		if (atomic_long_read(&calc_load_tasks_deferred)) {
+			deferred = atomic_long_xchg(&calc_load_tasks_deferred,
+						    0);
+			delta += deferred;
+		}
+
 		atomic_long_add(delta, &calc_load_tasks);
 	}
 }
@@ -3106,8 +3126,8 @@ static void update_cpu_load(struct rq *this_rq)
 	}
 
 	if (time_after_eq(jiffies, this_rq->calc_load_update)) {
-		this_rq->calc_load_update += LOAD_FREQ;
 		calc_load_account_active(this_rq);
+		this_rq->calc_load_update += LOAD_FREQ;
 	}
 }
 
Signature · 1 line
1.6.3.3
#1 lore raw
Quoted 14 lines · Chase Douglas

There's a period of 10 ticks where calc_load_tasks is updated by all the cpus for the load avg. Usually all the cpus do this during the first tick. If any cpus go idle, calc_load_tasks is decremented accordingly. However, if they wake up calc_load_tasks is not incremented. Thus, if cpus go idle during the 10 tick period, calc_load_tasks may be decremented to a non-representative value. This issue can lead to systems having a load avg of exactly 0, even though the real load avg could theoretically be up to NR_CPUS.

This change defers calc_load_tasks accounting after each cpu updates the count until after the 10 tick update window.

A few points:

Quoted 8 lines

* A global atomic deferral counter, and not per-cpu vars, is needed because a cpu may go NOHZ idle and not be able to update the global calc_load_tasks variable for subsequent load calculations.
* It is not enough to add calls to account for the load when a cpu is awakened:
- Load avg calculation must be independent of cpu load.
- If a cpu is awakend by one tasks, but then has more scheduled before the end of the update window, only the first task will be accounted.

OK, so what you're saying is that because we update calc_load_tasks from entering idle, we decrease earlier than a regular 10 tick sample interval would?

Hence you batch these early updates into _deferred and let the next 10 tick sample roll them over?

So the only early updates can come from pick_next_task_idle()->calc_load_account_active(), so why not specialize that callchain instead of the below?

Also, since its all NO_HZ, why not stick this in with the ILB? Once people get around to making that scale better, this can hitch a ride.

Something like the below perhaps? It does run partially from softirq context, but since there's a distinct lack of synchronization here that didn't seem like an immediate problem.

---
 kernel/sched.c          |   10 ++++++----
 kernel/sched_fair.c     |    4 +++-
 kernel/sched_idletask.c |    2 --
 3 files changed, 9 insertions(+), 7 deletions(-)
kernel/sched.c +6 -4
index 95eaecc..cdd4d8c 100644
@@ -2959,6 +2959,11 @@ static void calc_load_account_active(struct rq *this_rq)
 {
 	long nr_active, delta;
 
+	if (!time_after_eq(jiffies, this_rq->calc_load_update))
+		return;
+
+	this_rq->calc_load_update += LOAD_FREQ;
+
 	nr_active = this_rq->nr_running;
 	nr_active += (long) this_rq->nr_uninterruptible;
 
@@ -2998,10 +3003,7 @@ static void update_cpu_load(struct rq *this_rq)
 		this_rq->cpu_load[i] = (old_load*(scale-1) + new_load) >> i;
 	}
 
-	if (time_after_eq(jiffies, this_rq->calc_load_update)) {
-		this_rq->calc_load_update += LOAD_FREQ;
-		calc_load_account_active(this_rq);
-	}
+	calc_load_account_active(this_rq);
 }
 
 #ifdef CONFIG_SMP
kernel/sched_fair.c +3 -1
index 88d3053..2c267ef 100644
@@ -3394,9 +3394,11 @@ static void run_rebalance_domains(struct softirq_action *h)
 			if (need_resched())
 				break;
 
+			rq = cpu_rq(balance_cpu);
+			calc_load_account_active(rq);
+
 			rebalance_domains(balance_cpu, CPU_IDLE);
 
-			rq = cpu_rq(balance_cpu);
 			if (time_after(this_rq->next_balance, rq->next_balance))
 				this_rq->next_balance = rq->next_balance;
 		}
kernel/sched_idletask.c +0 -2
index bea2b8f..6ca191f 100644
@@ -23,8 +23,6 @@ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int fl
 static struct task_struct *pick_next_task_idle(struct rq *rq)
 {
 	schedstat_inc(rq, sched_goidle);
-	/* adjust the active tasks as we might go into a long sleep */
-	calc_load_account_active(rq);
 	return rq->idle;
 }
 
#2 lore raw
Quoted 4 lines · Peter Zijlstra

So the only early updates can come from pick_next_task_idle()->calc_load_account_active(), so why not specialize that callchain instead of the below?

To clarify, when I wrote that your patch was still below.. ;-)

#3 lore raw
In reply to Peter Zijlstra
Quoted 24 lines · Peter Zijlstra
Quoted 14 lines · Chase Douglas

There's a period of 10 ticks where calc_load_tasks is updated by all the cpus for the load avg. Usually all the cpus do this during the first tick. If any cpus go idle, calc_load_tasks is decremented accordingly. However, if they wake up calc_load_tasks is not incremented. Thus, if cpus go idle during the 10 tick period, calc_load_tasks may be decremented to a non-representative value. This issue can lead to systems having a load avg of exactly 0, even though the real load avg could theoretically be up to NR_CPUS.

This change defers calc_load_tasks accounting after each cpu updates the count until after the 10 tick update window.

A few points:

Quoted 8 lines

* A global atomic deferral counter, and not per-cpu vars, is needed because a cpu may go NOHZ idle and not be able to update the global calc_load_tasks variable for subsequent load calculations.
* It is not enough to add calls to account for the load when a cpu is awakened:
  - Load avg calculation must be independent of cpu load.
  - If a cpu is awakend by one tasks, but then has more scheduled before the end of the update window, only the first task will be accounted.

Quoted 6 lines

OK, so what you're saying is that because we update calc_load_tasks from entering idle, we decrease earlier than a regular 10 tick sample interval would?

Hence you batch these early updates into _deferred and let the next 10 tick sample roll them over?

Correct

Quoted 10 lines

So the only early updates can come from pick_next_task_idle()->calc_load_account_active(), so why not specialize that callchain instead of the below?

Also, since its all NO_HZ, why not stick this in with the ILB? Once people get around to making that scale better, this can hitch a ride.

Something like the below perhaps? It does run partially from softirq context, but since there's a distinct lack of synchronization here that didn't seem like an immediate problem.

I understand everything until you move the calc_load_account_active call to run_rebalance_domains. I take it that when CPUs go NO_HZ idle, at least one cpu is left to monitor and perform updates as necessary. Conceptually, it makes sense that this cpu should be handling the load accounting updates. However, I'm new to this code, so I'm having a hard time understanding all the cases and timings for when the scheduler softirq is called. Is it guaranteed to be called during every 10 tick load update window? If not, then we'll have the issue where a NO_HZ idle cpu won't be updated to 0 running tasks in time for the load avg calculation.

Would someone be able to explain how we are guaranteed of the correct timing for this path?

I also have a concern with run_rebalance_domains: If the designated no_hz.load_balancer cpu wasn't idle at the last tick or needs rescheduling, load accounting won't occur for idle cpus. Is it possible for this to occur every time when called in the 10 tick update window?

-- Chase

#4 lore raw

(stripped the ubuntu kernel-team list, since it is generating bounces for each email)

Quoted 10 lines · Chase Douglas
Quoted 6 lines

Also, since its all NO_HZ, why not stick this in with the ILB? Once people get around to making that scale better, this can hitch a ride.

Something like the below perhaps? It does run partially from softirq context, but since there's a distinct lack of synchronization here that didn't seem like an immediate problem.

I understand everything until you move the calc_load_account_active call to run_rebalance_domains. I take it that when CPUs go NO_HZ idle, at least one cpu is left to monitor and perform updates as necessary.

Right, that is the idea.

Quoted 7 lines

Conceptually, it makes sense that this cpu should be handling the load accounting updates. However, I'm new to this code, so I'm having a hard time understanding all the cases and timings for when the scheduler softirq is called. Is it guaranteed to be called during every 10 tick load update window? If not, then we'll have the issue where a NO_HZ idle cpu won't be updated to 0 running tasks in time for the load avg calculation.

Ah, I overlooked that trigger_load_balance() already has a jiffy delay. I was ass-uming we triggered the softirq on each tick.

Yes, that needs a bit of a fix to get called at least every 10 ticks, looking at rebalance_domain() that can end up being 60s.

Quoted 8 lines

Would someone be able to explain how we are guaranted of the correct timing for this path?

I also have a concern with run_rebalance_domains: If the designated no_hz.load_balancer cpu wasn't idle at the last tick or needs rescheduling, load accounting won't occur for idle cpus. Is it possible for this to occur every time when called in the 10 tick update window?

Right, so I didn't look too closely either, but was more or less going for the structure than the details. I haven't read through the ILB stuff in a while.

Quoted 1 line

From what I can quickly see, trigger_load_balance() will check if the

current cpu is idle_at_tick, if not it will nominate another cpu to be ilb -- so I guess it neatly fits together, the !idle cpus fend for themselves and the idle ones get sorted by the ILB.

Alternatively you could add the calc_load_tasks_deferred thing to the nohz structure and do it all from trigger_load_balance.

Either approach would work if the ILB were extended to be per node or something like that (Venki used to work on that, not sure what happened to that).

#5 lore raw
Quoted 47 lines · Chase Douglas
Quoted 11 lines
Quoted 7 lines

Also, since its all NO_HZ, why not stick this in with the ILB? Once people get around to making that scale better, this can hitch a ride.

Something like the below perhaps? It does run partially from softirq context, but since there's a distinct lack of synchronization here that didn't seem like an immediate problem.

I understand everything until you move the calc_load_account_active call to run_rebalance_domains. I take it that when CPUs go NO_HZ idle, at least one cpu is left to monitor and perform updates as necessary.

Right, that is the idea.

Quoted 7 lines

Conceptually, it makes sense that this cpu should be handling the load accounting updates. However, I'm new to this code, so I'm having a hard time understanding all the cases and timings for when the scheduler softirq is called. Is it guaranteed to be called during every 10 tick load update window? If not, then we'll have the issue where a NO_HZ idle cpu won't be updated to 0 running tasks in time for the load avg calculation.

Ah, I overlooked that trigger_load_balance() already has a jiffy delay. I was ass-uming we triggered the softirq on each tick.

Yes, that needs a bit of a fix to get called at least every 10 ticks, looking at rebalance_domain() that can end up being 60s.

Quoted 8 lines

Would someone be able to explain how we are guaranted of the correct timing for this path?

I also have a concern with run_rebalance_domains: If the designated no_hz.load_balancer cpu wasn't idle at the last tick or needs rescheduling, load accounting won't occur for idle cpus. Is it possible for this to occur every time when called in the 10 tick update window?

Right, so I didn't look too closely either, but was more or less going for the structure than the details. I haven't read through the ILB stuff in a while.

Quoted 1 line

From what I can quickly see, trigger_load_balance() will check if the

current cpu is idle_at_tick, if not it will nominate another cpu to be ilb -- so I guess it neatly fits together, the !idle cpus fend for themselves and the idle ones get sorted by the ILB.

Quoted 6 lines

Alternatively you could add the calc_load_tasks_deferred thing to the nohz structure and do it all from trigger_load_balance.

Either approach would work if the ILB were extended to be per node or something like that (Venki used to work on that, not sure what happened to that).

I really don't feel comfortable with this code to know how to implement either of these approaches myself. I have no issue how the fix is implemented. However, I worry that using the ILB code may be complex and/or require many little checks to ensure there are no improper interactions. Will we be sure that further changes to the ILB won't introduce new issues? In the end, what do we gain by using the ILB, and is it worth it to introduce that complexity?

-- Chase

#6 lore raw
In reply to Chase Douglas
Quoted 15 lines · Chase Douglas

There's a period of 10 ticks where calc_load_tasks is updated by all the cpus for the load avg. Usually all the cpus do this during the first tick. If any cpus go idle, calc_load_tasks is decremented accordingly. However, if they wake up calc_load_tasks is not incremented. Thus, if cpus go idle during the 10 tick period, calc_load_tasks may be decremented to a non-representative value. This issue can lead to systems having a load avg of exactly 0, even though the real load avg could theoretically be up to NR_CPUS.

This change defers calc_load_tasks accounting after each cpu updates the count until after the 10 tick update window.

A few points:

Quoted 9 lines

* A global atomic deferral counter, and not per-cpu vars, is needed because a cpu may go NOHZ idle and not be able to update the global calc_load_tasks variable for subsequent load calculations.
* It is not enough to add calls to account for the load when a cpu is awakened:
- Load avg calculation must be independent of cpu load.
- If a cpu is awakend by one tasks, but then has more scheduled before the end of the update window, only the first task will be accounted.

Ok, so delaying the whole ILB angle for now, the below is a similar approach to yours but with a more explicit code flow.

Does that work for you?

---
 kernel/sched.c          |   80 +++++++++++++++++++++++++++++++++++++++-------
 kernel/sched_idletask.c |    3 +-
 2 files changed, 68 insertions(+), 15 deletions(-)
kernel/sched.c +67 -13
index 95eaecc..8ac02a9 100644
@@ -1815,7 +1815,7 @@ static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares)
 }
 #endif
 
-static void calc_load_account_active(struct rq *this_rq);
+static void calc_load_account_idle(struct rq *this_rq);
 static void update_sysctl(void);
 static int get_update_sysctl_factor(void);
 
@@ -2907,6 +2907,61 @@ static unsigned long calc_load_update;
 unsigned long avenrun[3];
 EXPORT_SYMBOL(avenrun);
 
+static long calc_load_fold_active(struct rq *this_rq)
+{
+	long nr_active, delta = 0;
+
+	nr_active = this_rq->nr_running;
+	nr_active += (long) this_rq->nr_uninterruptible;
+
+	if (nr_active != this_rq->calc_load_active) {
+		delta = nr_active - this_rq->calc_load_active;
+		this_rq->calc_load_active = nr_active;
+	}
+
+	return delta;
+}
+
+#ifdef CONFIG_NO_HZ
+/*
+ * For NO_HZ we delay the active fold to the next LOAD_FREQ update.
+ *
+ * When making the ILB scale, we should try to pull this in as well.
+ */
+static atomic_long_t calc_load_tasks_idle;
+
+static void calc_load_account_idle(struct rq *this_rq)
+{
+	long delta;
+       
+	delta = calc_load_fold_active(this_rq);
+	if (delta)
+		atomic_long_add(delta, &calc_load_tasks_idle);
+}
+
+static long calc_load_fold_idle(void)
+{
+	long delta = 0;
+
+	/*
+	 * Its got a race, we don't care...
+	 */
+	if (atomic_long_read(&calc_load_tasks_idle))
+		delta = atomic_long_xchg(&calc_load_tasks_idle, 0);
+
+	return delta;
+}
+#else
+static void calc_load_account_idle(struct rq *this_rq)
+{
+}
+
+static inline long calc_load_fold_idle(void)
+{
+	return 0;
+}
+#endif
+
 /**
  * get_avenrun - get the load average array
  * @loads:	pointer to dest load array
@@ -2953,20 +3008,22 @@ void calc_global_load(void)
 }
 
 /*
- * Either called from update_cpu_load() or from a cpu going idle
+ * Called from update_cpu_load() to periodically update this CPU's
+ * active count.
  */
 static void calc_load_account_active(struct rq *this_rq)
 {
-	long nr_active, delta;
+	long delta;
 
-	nr_active = this_rq->nr_running;
-	nr_active += (long) this_rq->nr_uninterruptible;
+	if (time_before(jiffies, this_rq->calc_load_update))
+		return;
 
-	if (nr_active != this_rq->calc_load_active) {
-		delta = nr_active - this_rq->calc_load_active;
-		this_rq->calc_load_active = nr_active;
+	delta  = calc_load_fold_active(this_rq);
+	delta += calc_load_fold_idle();
+	if (delta)
 		atomic_long_add(delta, &calc_load_tasks);
-	}
+
+	this_rq->calc_load_update += LOAD_FREQ;
 }
 
 /*
@@ -2998,10 +3055,7 @@ static void update_cpu_load(struct rq *this_rq)
 		this_rq->cpu_load[i] = (old_load*(scale-1) + new_load) >> i;
 	}
 
-	if (time_after_eq(jiffies, this_rq->calc_load_update)) {
-		this_rq->calc_load_update += LOAD_FREQ;
-		calc_load_account_active(this_rq);
-	}
+	calc_load_account_active(this_rq);
 }
 
 #ifdef CONFIG_SMP
kernel/sched_idletask.c +1 -2
index bea2b8f..9fa0f40 100644
@@ -23,8 +23,7 @@ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int fl
 static struct task_struct *pick_next_task_idle(struct rq *rq)
 {
 	schedstat_inc(rq, sched_goidle);
-	/* adjust the active tasks as we might go into a long sleep */
-	calc_load_account_active(rq);
+	calc_load_account_idle(rq);
 	return rq->idle;
 }
 
#7 lore raw
Quoted 26 lines · Peter Zijlstra
Quoted 15 lines · Chase Douglas

There's a period of 10 ticks where calc_load_tasks is updated by all the cpus for the load avg. Usually all the cpus do this during the first tick. If any cpus go idle, calc_load_tasks is decremented accordingly. However, if they wake up calc_load_tasks is not incremented. Thus, if cpus go idle during the 10 tick period, calc_load_tasks may be decremented to a non-representative value. This issue can lead to systems having a load avg of exactly 0, even though the real load avg could theoretically be up to NR_CPUS.

This change defers calc_load_tasks accounting after each cpu updates the count until after the 10 tick update window.

A few points:

Quoted 9 lines

* A global atomic deferral counter, and not per-cpu vars, is needed because a cpu may go NOHZ idle and not be able to update the global calc_load_tasks variable for subsequent load calculations.
* It is not enough to add calls to account for the load when a cpu is awakened:
  - Load avg calculation must be independent of cpu load.
  - If a cpu is awakend by one tasks, but then has more scheduled before the end of the update window, only the first task will be accounted.

Quoted 4 lines

Ok, so delaying the whole ILB angle for now, the below is a similar approach to yours but with a more explicit code flow.

Does that work for you?

This looks good. I'll run my test case to make sure it fixes the scenario we hit, and then I'll ack it when I've confirmed it works.

Thanks

-- Chase

author Acked-by
#8 lore raw
Quoted 27 lines · Chase Douglas
Quoted 26 lines · Peter Zijlstra
Quoted 24 lines · Chase Douglas

There's a period of 10 ticks where calc_load_tasks is updated by all the cpus for the load avg. Usually all the cpus do this during the first tick. If any cpus go idle, calc_load_tasks is decremented accordingly. However, if they wake up calc_load_tasks is not incremented. Thus, if cpus go idle during the 10 tick period, calc_load_tasks may be decremented to a non-representative value. This issue can lead to systems having a load avg of exactly 0, even though the real load avg could theoretically be up to NR_CPUS.

This change defers calc_load_tasks accounting after each cpu updates the count until after the 10 tick update window.

A few points:

* A global atomic deferral counter, and not per-cpu vars, is needed because a cpu may go NOHZ idle and not be able to update the global calc_load_tasks variable for subsequent load calculations.
* It is not enough to add calls to account for the load when a cpu is awakened:
  - Load avg calculation must be independent of cpu load.
  - If a cpu is awakend by one tasks, but then has more scheduled before the end of the update window, only the first task will be accounted.

Quoted 7 lines
Quoted 4 lines

Ok, so delaying the whole ILB angle for now, the below is a similar approach to yours but with a more explicit code flow.

Does that work for you?

This looks good. I'll run my test case to make sure it fixes the scenario we hit, and then I'll ack it when I've confirmed it works.

I've run my test case and it seems to push the load avg numbers as expected.

BTW, I noticed some trailing whitespace, so I ran it through checkpatch.pl:

ERROR: trailing whitespace
#44: FILE: kernel/sched.c:2936:
+ $

Thanks

-- Chase

#9 lore raw

[tip:sched/core] sched: Cure load average vs NO_HZ woes

In reply to Peter Zijlstra
Commit-ID:  74f5187ac873042f502227701ed1727e7c5fbfa9
Gitweb:     http://git.kernel.org/tip/74f5187ac873042f502227701ed1727e7c5fbfa9
Author:     Peter Zijlstra <[email protected]>
AuthorDate: Thu, 22 Apr 2010 21:50:19 +0200
Committer:  Ingo Molnar <[email protected]>
CommitDate: Fri, 23 Apr 2010 11:02:02 +0200

sched: Cure load average vs NO_HZ woes

Chase reported that due to us decrementing calc_load_task prematurely (before the next LOAD_FREQ sample), the load average could be scewed by as much as the number of CPUs in the machine.

This patch, based on Chase's patch, cures the problem by keeping the delta of the CPU going into NO_HZ idle separately and folding that in on the next LOAD_FREQ update.

This restores the balance and we get strict LOAD_FREQ period samples.

LKML-Reference: <1271934490.1776.343.camel@laptop>

---
 kernel/sched.c          |   80 +++++++++++++++++++++++++++++++++++++++-------
 kernel/sched_idletask.c |    3 +-
 2 files changed, 68 insertions(+), 15 deletions(-)
kernel/sched.c +67 -13
index de0da71..0cc913a 100644
@@ -1815,7 +1815,7 @@ static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares)
 }
 #endif
 
-static void calc_load_account_active(struct rq *this_rq);
+static void calc_load_account_idle(struct rq *this_rq);
 static void update_sysctl(void);
 static int get_update_sysctl_factor(void);
 
@@ -2950,6 +2950,61 @@ static unsigned long calc_load_update;
 unsigned long avenrun[3];
 EXPORT_SYMBOL(avenrun);
 
+static long calc_load_fold_active(struct rq *this_rq)
+{
+	long nr_active, delta = 0;
+
+	nr_active = this_rq->nr_running;
+	nr_active += (long) this_rq->nr_uninterruptible;
+
+	if (nr_active != this_rq->calc_load_active) {
+		delta = nr_active - this_rq->calc_load_active;
+		this_rq->calc_load_active = nr_active;
+	}
+
+	return delta;
+}
+
+#ifdef CONFIG_NO_HZ
+/*
+ * For NO_HZ we delay the active fold to the next LOAD_FREQ update.
+ *
+ * When making the ILB scale, we should try to pull this in as well.
+ */
+static atomic_long_t calc_load_tasks_idle;
+
+static void calc_load_account_idle(struct rq *this_rq)
+{
+	long delta;
+
+	delta = calc_load_fold_active(this_rq);
+	if (delta)
+		atomic_long_add(delta, &calc_load_tasks_idle);
+}
+
+static long calc_load_fold_idle(void)
+{
+	long delta = 0;
+
+	/*
+	 * Its got a race, we don't care...
+	 */
+	if (atomic_long_read(&calc_load_tasks_idle))
+		delta = atomic_long_xchg(&calc_load_tasks_idle, 0);
+
+	return delta;
+}
+#else
+static void calc_load_account_idle(struct rq *this_rq)
+{
+}
+
+static inline long calc_load_fold_idle(void)
+{
+	return 0;
+}
+#endif
+
 /**
  * get_avenrun - get the load average array
  * @loads:	pointer to dest load array
@@ -2996,20 +3051,22 @@ void calc_global_load(void)
 }
 
 /*
- * Either called from update_cpu_load() or from a cpu going idle
+ * Called from update_cpu_load() to periodically update this CPU's
+ * active count.
  */
 static void calc_load_account_active(struct rq *this_rq)
 {
-	long nr_active, delta;
+	long delta;
 
-	nr_active = this_rq->nr_running;
-	nr_active += (long) this_rq->nr_uninterruptible;
+	if (time_before(jiffies, this_rq->calc_load_update))
+		return;
 
-	if (nr_active != this_rq->calc_load_active) {
-		delta = nr_active - this_rq->calc_load_active;
-		this_rq->calc_load_active = nr_active;
+	delta  = calc_load_fold_active(this_rq);
+	delta += calc_load_fold_idle();
+	if (delta)
 		atomic_long_add(delta, &calc_load_tasks);
-	}
+
+	this_rq->calc_load_update += LOAD_FREQ;
 }
 
 /*
@@ -3041,10 +3098,7 @@ static void update_cpu_load(struct rq *this_rq)
 		this_rq->cpu_load[i] = (old_load*(scale-1) + new_load) >> i;
 	}
 
-	if (time_after_eq(jiffies, this_rq->calc_load_update)) {
-		this_rq->calc_load_update += LOAD_FREQ;
-		calc_load_account_active(this_rq);
-	}
+	calc_load_account_active(this_rq);
 }
 
 #ifdef CONFIG_SMP
kernel/sched_idletask.c +1 -2
index bea2b8f..9fa0f40 100644
@@ -23,8 +23,7 @@ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int fl
 static struct task_struct *pick_next_task_idle(struct rq *rq)
 {
 	schedstat_inc(rq, sched_goidle);
-	/* adjust the active tasks as we might go into a long sleep */
-	calc_load_account_active(rq);
+	calc_load_account_idle(rq);
 	return rq->idle;
 }