sched/fair: Make cfs_rq::decay_counter non-atomic

We update decay_counter in update_cfs_rq_blocked_load() only. This function is always called with rq lock locked, so we can kill atomic actions.

Signed-off-by Kirill Tkhai <[email protected]>

---
 kernel/sched/fair.c  |   11 ++++++-----
 kernel/sched/sched.h |    2 +-
 2 files changed, 7 insertions(+), 6 deletions(-)

kernel/sched/fair.c +6 -5

index 5f3b5a7..af990c4 100644
@@ -2570,7 +2570,7 @@ static __always_inline int __update_entity_runnable_avg(u64 now,
 static inline u64 __synchronize_entity_decay(struct sched_entity *se)
 {
 	struct cfs_rq *cfs_rq = cfs_rq_of(se);
-	u64 decays = atomic64_read(&cfs_rq->decay_counter);
+	u64 decays = ACCESS_ONCE(cfs_rq->decay_counter);
 
 	decays -= se->avg.decay_count;
 	if (!decays)

@@ -2767,7 +2767,8 @@ static void update_cfs_rq_blocked_load(struct cfs_rq *cfs_rq, int force_update)
 	if (decays) {
 		cfs_rq->blocked_load_avg = decay_load(cfs_rq->blocked_load_avg,
 						      decays);
-		atomic64_add(decays, &cfs_rq->decay_counter);
+		lockdep_assert_held(&rq_of(cfs_rq)->lock);
+		cfs_rq->decay_counter += decays;
 		cfs_rq->last_decay = now;
 	}
 

@@ -2837,7 +2838,7 @@ static inline void dequeue_entity_load_avg(struct cfs_rq *cfs_rq,
 	cfs_rq->runnable_load_avg -= se->avg.load_avg_contrib;
 	if (sleep) {
 		cfs_rq->blocked_load_avg += se->avg.load_avg_contrib;
-		se->avg.decay_count = atomic64_read(&cfs_rq->decay_counter);
+		se->avg.decay_count = ACCESS_ONCE(cfs_rq->decay_counter);
 	} /* migrations, e.g. sleep=0 leave decay_count == 0 */
 }
 

@@ -7876,7 +7877,7 @@ void init_cfs_rq(struct cfs_rq *cfs_rq)
 	cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime;
 #endif
 #ifdef CONFIG_SMP
-	atomic64_set(&cfs_rq->decay_counter, 1);
+	cfs_rq->decay_counter = 1;
 	atomic_long_set(&cfs_rq->removed_load, 0);
 #endif
 }

@@ -7928,7 +7929,7 @@ static void task_move_group_fair(struct task_struct *p, int queued)
 		 * contribution, but we must synchronize for ongoing future
 		 * decay.
 		 */
-		se->avg.decay_count = atomic64_read(&cfs_rq->decay_counter);
+		se->avg.decay_count = ACCESS_ONCE(cfs_rq->decay_counter);
 		cfs_rq->blocked_load_avg += se->avg.load_avg_contrib;
 #endif
 	}

kernel/sched/sched.h +1 -1

index 9a2a45c..672f120 100644
@@ -364,7 +364,7 @@ struct cfs_rq {
 	 * the FAIR_GROUP_SCHED case).
 	 */
 	unsigned long runnable_load_avg, blocked_load_avg;
-	atomic64_t decay_counter;
+	u64 decay_counter;
 	u64 last_decay;
 	atomic_long_t removed_load;
 

We update decay_counter in update_cfs_rq_blocked_load() only. This function is always called with rq lock locked, so we can kill atomic actions.

Signed-off-by Kirill Tkhai <[email protected]>

---
 kernel/sched/fair.c  |   11 ++++++-----
 kernel/sched/sched.h |    2 +-
 2 files changed, 7 insertions(+), 6 deletions(-)

kernel/sched/fair.c +0 -0

index 5f3b5a7..af990c4 100644

Quoted 10 lines · Kirill Tkhai

We update decay_counter in update_cfs_rq_blocked_load() only. This function is always called with rq lock locked, so we can kill atomic actions.

Signed-off-by Kirill Tkhai <[email protected]>

---
 kernel/sched/fair.c  |   11 ++++++-----
 kernel/sched/sched.h |    2 +-
 2 files changed, 7 insertions(+), 6 deletions(-)

kernel/sched/fair.c static __always_inline int __update_entity_runnable_avg(u64 now, · line 2573

Quoted 10 lines

kernel/sched/fair.c +1 -1

index 5f3b5a7..af990c4 100644
@@ -2570,7 +2570,7 @@ static __always_inline int __update_entity_runnable_avg(u64 now,
 static inline u64 __synchronize_entity_decay(struct sched_entity *se)
 {
 	struct cfs_rq *cfs_rq = cfs_rq_of(se);
-	u64 decays = atomic64_read(&cfs_rq->decay_counter);
+	u64 decays = ACCESS_ONCE(cfs_rq->decay_counter);

This is called without rq lock held from migrate_task_rq_fair. (We could technically change the atomic_add to atomic64_set(atomic64_read() + x), but I don't know that that is a win) Now, we could do a min_vruntime-style two-copy thing if this atomic usage is a hot spot on 32-bit, we just didn't bother initially.