sched/debug: decouple sched_stat tracepoints from CONFIG_SCHEDSTATS
Subsystems (2)
cover sched/debug: decouple sched_stat tracepoints from CONFIG_SCHEDSTATS
NOTE: I didn't include any performance numbers because I wasn't able to get consistent results. I tried the following on a Xeon E5-2420 v2 CPU:
$ for i in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo -n performance > $i; done $ echo 1 > /sys/devices/system/cpu/intel_pstate/no_turbo $ echo 100 > /sys/devices/system/cpu/intel_pstate/min_perf_pct $ echo 0 > /proc/sys/kernel/nmi_watchdog $ taskset 0x10 perf stat -n -r10 perf bench sched pipe -l 1000000
I was going to post the numbers from that, both with and without SCHEDSTATS, but then when I tried to repeat the test on a different day, the results were surprisingly different, with different conclusions.
So any advice on measuring scheduler performance would be appreciated...
Josh Poimboeuf (5):
sched/debug: rename and move enqueue_sleeper() sched/debug: schedstat macro cleanup
sched/debug: 'schedstat_val()' -> 'schedstat_val_or_zero()' sched/debug: remove several CONFIG_SCHEDSTATS guards sched/debug: decouple 'sched_stat_*' tracepoints' from
CONFIG_SCHEDSTATS
include/linux/sched.h | 11 +- kernel/latencytop.c | 2 - kernel/profile.c | 5 - kernel/sched/core.c | 59 ++++------ kernel/sched/debug.c | 104 +++++++++-------- kernel/sched/fair.c | 290 ++++++++++++++++++++--------------------------- kernel/sched/idle_task.c | 2 +- kernel/sched/stats.h | 24 ++-- lib/Kconfig.debug | 1 - 9 files changed, 220 insertions(+), 278 deletions(-)
Signature · 1 line
2.4.11
PATCH 1/5 sched/debug: rename and move enqueue_sleeper()
enqueue_sleeper() doesn't actually enqueue, it just handles some statistics and tracepoints. Rename it to update_stats_enqueue_sleeper() and call it from update_stats_enqueue().
--- kernel/sched/fair.c | 142 +++++++++++++++++++++++++++------------------------- 1 file changed, 73 insertions(+), 69 deletions(-)
kernel/sched/fair.c +73 -69
@@ -830,11 +830,72 @@ update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se) se->statistics.wait_start =0 ; } +static void +update_stats_enqueue_sleeper (struct cfs_rq *cfs_rq,struct sched_entity *se) +{ +struct task_struct *tsk =NULL ; + +if (entity_is_task (se)) + tsk =task_of (se); + +if (se->statistics.sleep_start) { +u64 delta =rq_clock (rq_of (cfs_rq)) - se->statistics.sleep_start; + +if ((s64 )delta <0 ) + delta =0 ; + +if (unlikely (delta > se->statistics.sleep_max)) + se->statistics.sleep_max = delta; + + se->statistics.sleep_start =0 ; + se->statistics.sum_sleep_runtime += delta; + +if (tsk) { +account_scheduler_latency (tsk, delta >>10 ,1 ); +trace_sched_stat_sleep (tsk, delta); + } + } +if (se->statistics.block_start) { +u64 delta =rq_clock (rq_of (cfs_rq)) - se->statistics.block_start; + +if ((s64 )delta <0 ) + delta =0 ; + +if (unlikely (delta > se->statistics.block_max)) + se->statistics.block_max = delta; + + se->statistics.block_start =0 ; + se->statistics.sum_sleep_runtime += delta; + +if (tsk) { +if (tsk->in_iowait) { + se->statistics.iowait_sum += delta; + se->statistics.iowait_count++; +trace_sched_stat_iowait (tsk, delta); + } + +trace_sched_stat_blocked (tsk, delta); + +/* +* Blocking time is in units of nanosecs, so shift by +* 20 to get a milliseconds-range estimation of the +* amount of time that the task spent sleeping: +*/ +if (unlikely (prof_on ==SLEEP_PROFILING )) { +profile_hits (SLEEP_PROFILING , + (void *)get_wchan (tsk), + delta >>20 ); + } +account_scheduler_latency (tsk, delta >>10 ,0 ); + } + } +} +/* * Task is being enqueued - update stats: */ static inline void -update_stats_enqueue (struct cfs_rq *cfs_rq,struct sched_entity *se) +update_stats_enqueue (struct cfs_rq *cfs_rq,struct sched_entity *se,int flags) {/* * Are we enqueueing a waiting task? (for current tasks
@@ -842,6 +903,9 @@ update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)*/ if (se != cfs_rq->curr)update_stats_wait_start (cfs_rq, se); + +if (flags &ENQUEUE_WAKEUP ) +update_stats_enqueue_sleeper (cfs_rq, se); } static inline void
@@ -878,7 +942,12 @@ update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se) }static inline void -update_stats_enqueue (struct cfs_rq *cfs_rq,struct sched_entity *se) +update_stats_enqueue_sleeper (struct cfs_rq *cfs_rq,struct sched_entity *se) +{ +} + +static inline void +update_stats_enqueue (struct cfs_rq *cfs_rq,struct sched_entity *se,int flags) { }
@@ -3099,68 +3168,6 @@ static inline int idle_balance(struct rq *rq)#endif /* CONFIG_SMP */ -static void enqueue_sleeper (struct cfs_rq *cfs_rq,struct sched_entity *se) -{ -#ifdef CONFIG_SCHEDSTATS -struct task_struct *tsk =NULL ; - -if (entity_is_task (se)) - tsk =task_of (se); - -if (se->statistics.sleep_start) { -u64 delta =rq_clock (rq_of (cfs_rq)) - se->statistics.sleep_start; - -if ((s64 )delta <0 ) - delta =0 ; - -if (unlikely (delta > se->statistics.sleep_max)) - se->statistics.sleep_max = delta; - - se->statistics.sleep_start =0 ; - se->statistics.sum_sleep_runtime += delta; - -if (tsk) { -account_scheduler_latency (tsk, delta >>10 ,1 ); -trace_sched_stat_sleep (tsk, delta); - } - } -if (se->statistics.block_start) { -u64 delta =rq_clock (rq_of (cfs_rq)) - se->statistics.block_start; - -if ((s64 )delta <0 ) - delta =0 ; - -if (unlikely (delta > se->statistics.block_max)) - se->statistics.block_max = delta; - - se->statistics.block_start =0 ; - se->statistics.sum_sleep_runtime += delta; - -if (tsk) { -if (tsk->in_iowait) { - se->statistics.iowait_sum += delta; - se->statistics.iowait_count++; -trace_sched_stat_iowait (tsk, delta); - } - -trace_sched_stat_blocked (tsk, delta); - -/* -* Blocking time is in units of nanosecs, so shift by -* 20 to get a milliseconds-range estimation of the -* amount of time that the task spent sleeping: -*/ -if (unlikely (prof_on ==SLEEP_PROFILING )) { -profile_hits (SLEEP_PROFILING , - (void *)get_wchan (tsk), - delta >>20 ); - } -account_scheduler_latency (tsk, delta >>10 ,0 ); - } - } -#endif -} -static void check_spread (struct cfs_rq *cfs_rq,struct sched_entity *se) {#ifdef CONFIG_SCHED_DEBUG
@@ -3287,15 +3294,12 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)account_entity_enqueue (cfs_rq, se);update_cfs_shares (cfs_rq); -if (flags &ENQUEUE_WAKEUP ) { +if (flags &ENQUEUE_WAKEUP )place_entity (cfs_rq, se,0 ); -if (schedstat_enabled ()) -enqueue_sleeper (cfs_rq, se); - }check_schedstat_required ();if (schedstat_enabled ()) { -update_stats_enqueue (cfs_rq, se); +update_stats_enqueue (cfs_rq, se, flags);check_spread (cfs_rq, se); }if (!curr)
Signature · 1 line
2.4.11
PATCH 4/5 sched/debug: remove several CONFIG_SCHEDSTATS guards
In reply to Josh PoimboeufClean up the sched code by removing several of the CONFIG_SCHEDSTATS guards, using schedstat_*() macros where needed.
Code size:
!CONFIG_SCHEDSTATS defconfig:
text data bss dec hex filename
10209818 4368184 1105920 15683922 ef5152 vmlinux.before.nostats
10209818 4368184 1105920 15683922 ef5152 vmlinux.after.nostats
CONFIG_SCHEDSTATS defconfig:
text data bss dec hex filename
10214210 4370040 1105920 15690170 ef69ba vmlinux.before.stats
10214210 4370680 1105920 15690810 ef6c3a vmlinux.after.stats
--- kernel/sched/core.c | 29 +++++----- kernel/sched/debug.c | 99 ++++++++++++++++++---------------- kernel/sched/fair.c | 148 ++++++++++++++++++++++++--------------------------- 3 files changed, 136 insertions(+), 140 deletions(-)
kernel/sched/core.c +14 -15
@@ -1626,10 +1626,15 @@ static inline int __set_cpus_allowed_ptr(struct task_struct *p,static void ttwu_stat (struct task_struct *p,int cpu,int wake_flags) { -#ifdef CONFIG_SCHEDSTATS - struct rq *rq = this_rq(); +struct rq *rq; + +if (!schedstat_enabled ()) +return ; + + rq =this_rq ();#ifdef CONFIG_SMP +{int this_cpu =smp_processor_id ();if (cpu == this_cpu) {
@@ -1651,7 +1656,7 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags)if (wake_flags &WF_MIGRATED ) schedstat_inc(p->se.statistics.nr_wakeups_migrate); - +}#endif /* CONFIG_SMP */ schedstat_inc (rq->ttwu_count);
@@ -1659,8 +1664,6 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags)if (wake_flags &WF_SYNC )schedstat_inc (p->se.statistics.nr_wakeups_sync); - -#endif /* CONFIG_SCHEDSTATS */ }static inline void ttwu_activate (struct rq *rq,struct task_struct *p,int en_flags)
@@ -2059,8 +2062,7 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)ttwu_queue (p, cpu, wake_flags); stat: -if (schedstat_enabled ()) -ttwu_stat (p, cpu, wake_flags); +ttwu_stat (p, cpu, wake_flags); out:raw_spin_unlock_irqrestore (&p->pi_lock, flags);
@@ -2108,8 +2110,7 @@ static void try_to_wake_up_local(struct task_struct *p, struct pin_cookie cookiettwu_activate (rq, p,ENQUEUE_WAKEUP );ttwu_do_wakeup (rq, p,0 , cookie); -if (schedstat_enabled ()) -ttwu_stat (p,smp_processor_id (),0 ); +ttwu_stat (p,smp_processor_id (),0 ); out:raw_spin_unlock (&p->pi_lock); }
@@ -7588,12 +7589,10 @@ void normalize_rt_tasks(void)if (p->flags &PF_KTHREAD )continue ; - p->se.exec_start =0 ; -#ifdef CONFIG_SCHEDSTATS - p->se.statistics.wait_start =0 ; - p->se.statistics.sleep_start =0 ; - p->se.statistics.block_start =0 ; -#endif + p->se.exec_start =0 ; +schedstat_set (p->se.statistics.wait_start,0 ); +schedstat_set (p->se.statistics.sleep_start,0 ); +schedstat_set (p->se.statistics.block_start,0 );if (!dl_task (p) && !rt_task (p)) {/*
kernel/sched/debug.c +53 -46
167 lines not on this page · raw message
kernel/sched/fair.c +69 -79
267 lines not on this page · raw message
Signature · 1 line
2.4.11
PATCH 5/5 sched/debug: decouple 'sched_stat_*' tracepoints' from CONFIG_SCHEDSTATS
In reply to Josh PoimboeufThe 'sched_stat_*' tracepoints have a dependency on schedstats being enabled at both compile-time (CONFIG_SCHEDSTATS) and runtime (sched_schedstats).
In addition to causing increased code complexity, it also causes confusion:
- When schedstats are disabled, the tracepoints don't fire at all.
- When schedstats are enabled at runtime, existing tasks will have stale or missing statistics, which can cause the tracepoints to either not fire, or to fire with corrupt data.
Decouple them so that the tracepoints will always fire correctly, independent of schedstats. A side benefit is that this also means that latencytop and profiling no longer have a dependency on schedstats.
Code size:
!CONFIG_SCHEDSTATS defconfig:
text data bss dec hex filename
10209818 4368184 1105920 15683922 ef5152 vmlinux.before.nostats
10209818 4368312 1105920 15684050 ef51d2 vmlinux.after.nostats CONFIG_SCHEDSTATS defconfig:
text data bss dec hex filename
10214210 4370680 1105920 15690810 ef6c3a vmlinux.before.stats
10214261 4370104 1105920 15690285 ef6a2d vmlinux.after.stats--- include/linux/sched.h | 11 +++---- kernel/latencytop.c | 2 -- kernel/profile.c | 5 ---- kernel/sched/core.c | 16 +++-------- kernel/sched/debug.c | 13 +++++---- kernel/sched/fair.c | 80 ++++++++++++--------------------------------------- lib/Kconfig.debug | 1 - 7 files changed, 34 insertions(+), 94 deletions(-)
include/linux/sched.h +4 -7
@@ -940,10 +940,6 @@ static inline int sched_info_on(void)#endif } -#ifdef CONFIG_SCHEDSTATS -void force_schedstat_enabled (void ); -#endif -enum cpu_idle_type {CPU_IDLE ,CPU_NOT_IDLE ,
@@ -1285,18 +1281,15 @@ struct sched_avg { #ifdef CONFIG_SCHEDSTATS struct sched_statistics { - u64 wait_start; u64 wait_max; u64 wait_count; u64 wait_sum; u64 iowait_count; u64 iowait_sum; - u64 sleep_start; u64 sleep_max; s64 sum_sleep_runtime; - u64 block_start; u64 block_max; u64 exec_max; u64 slice_max;
@@ -1332,6 +1325,10 @@ struct sched_entity {u64 nr_migrations; +u64 wait_start; +u64 sleep_start; +u64 block_start; +#ifdef CONFIG_SCHEDSTATS struct sched_statistics statistics;#endif
kernel/latencytop.c +0 -2
@@ -296,8 +296,6 @@ int sysctl_latencytop(struct ctl_table *table, int write,int err; err =proc_dointvec (table, write, buffer, lenp, ppos); -if (latencytop_enabled) -force_schedstat_enabled ();return err; }
kernel/profile.c +0 -5
@@ -58,8 +58,6 @@ int profile_setup(char *str)int par;if (!strncmp (str, sleepstr,strlen (sleepstr))) { -#ifdef CONFIG_SCHEDSTATS -force_schedstat_enabled (); prof_on =SLEEP_PROFILING ;if (str[strlen (sleepstr)] ==',' ) str +=strlen (sleepstr) +1 ;
@@ -67,9 +65,6 @@ int profile_setup(char *str) prof_shift = par;pr_info ("kernel sleep profiling enabled (shift: %ld)\n" , prof_shift); -#else -pr_warn ("kernel sleep profiling requires CONFIG_SCHEDSTATS\n" ); -#endif /* CONFIG_SCHEDSTATS */ }else if (!strncmp (str, schedstr,strlen (schedstr))) { prof_on =SCHED_PROFILING ;if (str[strlen (schedstr)] ==',' )
kernel/sched/core.c +4 -12
@@ -2267,14 +2267,6 @@ static void set_schedstats(bool enabled)static_branch_disable (&sched_schedstats); } -void force_schedstat_enabled (void ) -{ -if (!schedstat_enabled ()) { -pr_info ("kernel profiling enabled schedstats, disable via kernel.sched_schedstats.\n" ); -static_branch_enable (&sched_schedstats); - } -} -static int __init setup_schedstats (char *str) {int ret =0 ;
@@ -7589,10 +7581,10 @@ void normalize_rt_tasks(void)if (p->flags &PF_KTHREAD )continue ; - p->se.exec_start =0 ; -schedstat_set (p->se.statistics.wait_start,0 ); -schedstat_set (p->se.statistics.sleep_start,0 ); -schedstat_set (p->se.statistics.block_start,0 ); + p->se.exec_start =0 ; + p->se.wait_start =0 ; + p->se.sleep_start =0 ; + p->se.block_start =0 ;if (!dl_task (p) && !rt_task (p)) {/*
kernel/sched/debug.c +7 -6
@@ -382,10 +382,10 @@ static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_groupPN (se->exec_start);PN (se->vruntime);PN (se->sum_exec_runtime); +PN (se->wait_start); +PN (se->sleep_start); +PN (se->block_start);if (schedstat_enabled ()) { -PN_SCHEDSTAT (se->statistics.wait_start); -PN_SCHEDSTAT (se->statistics.sleep_start); -PN_SCHEDSTAT (se->statistics.block_start);PN_SCHEDSTAT (se->statistics.sleep_max);PN_SCHEDSTAT (se->statistics.block_max);PN_SCHEDSTAT (se->statistics.exec_max);
@@ -887,13 +887,14 @@ void proc_sched_show_task(struct task_struct *p, struct seq_file *m)P (se.nr_migrations); +PN (se.wait_start); +PN (se.sleep_start); +PN (se.block_start); +if (schedstat_enabled ()) {u64 avg_atom, avg_per_cpu;PN_SCHEDSTAT (se.statistics.sum_sleep_runtime); -PN_SCHEDSTAT (se.statistics.wait_start); -PN_SCHEDSTAT (se.statistics.sleep_start); -PN_SCHEDSTAT (se.statistics.block_start);PN_SCHEDSTAT (se.statistics.sleep_max);PN_SCHEDSTAT (se.statistics.block_max);PN_SCHEDSTAT (se.statistics.exec_max);
kernel/sched/fair.c +19 -61
171 lines not on this page · raw message
lib/Kconfig.debug +0 -1
@@ -1699,7 +1699,6 @@ config LATENCYTOP select KALLSYMS select KALLSYMS_ALL select STACKTRACE - select SCHEDSTATS select SCHED_DEBUG help Enable this option if you want to use the LatencyTOP tool
Signature · 1 line
2.4.11
PATCH 3/5 sched/debug: 'schedstat_val()' -> 'schedstat_val_or_zero()'
In reply to Josh PoimboeufThe schedstat_val() macro's behavior is kind of surprising: when schedstat is runtime disabled, it returns zero. Rename it to schedstat_val_or_zero().
There's also a need for a similar macro which doesn't have the 'if (schedstat_enable())' check, to avoid doing the check twice. Create a new 'schedstat_val()' macro for that.
--- kernel/sched/debug.c | 4 ++-- kernel/sched/stats.h | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-)
kernel/sched/debug.c +2 -2
@@ -429,9 +429,9 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) p->prio);SEQ_printf (m,"%9Ld.%06ld %9Ld.%06ld %9Ld.%06ld" , -SPLIT_NS (schedstat_val (p->se.statistics.wait_sum)), +SPLIT_NS (schedstat_val_or_zero (p->se.statistics.wait_sum)),SPLIT_NS (p->se.sum_exec_runtime), -SPLIT_NS (schedstat_val (p->se.statistics.sum_sleep_runtime))); +SPLIT_NS (schedstat_val_or_zero (p->se.statistics.sum_sleep_runtime)));#ifdef CONFIG_NUMA_BALANCING SEQ_printf (m," %d %d" ,task_node (p),task_numa_group_id (p));
kernel/sched/stats.h +3 -1
@@ -33,7 +33,8 @@ rq_sched_info_dequeued(struct rq *rq, unsigned long long delta)#define schedstat_inc (var) do { if (schedstat_enabled()) { var++; } } while (0)#define schedstat_add (var, amt) do { if (schedstat_enabled()) { var += (amt); } } while (0)#define schedstat_set (var, val) do { if (schedstat_enabled()) { var = (val); } } while (0) -#define schedstat_val (var) ((schedstat_enabled()) ? (var) : 0) +#define schedstat_val (var) (var) +#define schedstat_val_or_zero (var) ((schedstat_enabled()) ? (var) : 0)#else /* !CONFIG_SCHEDSTATS */ static inline void
@@ -50,6 +51,7 @@ rq_sched_info_depart(struct rq *rq, unsigned long long delta)#define schedstat_add (var, amt) do { } while (0)#define schedstat_set (var, val) do { } while (0)#define schedstat_val (var) 0 +#define schedstat_val_or_zero (var) 0#endif /* CONFIG_SCHEDSTATS */ #ifdef CONFIG_SCHED_INFO
Signature · 1 line
2.4.11
PATCH 2/5 sched/debug: schedstat macro cleanup
In reply to Josh PoimboeufThe schedstat_*() macros are inconsistent: most of them take a pointer and a field which the macro combines, whereas schedstat_set() takes the already combined ptr->field.
The already combined ptr->field argument is actually more intuitive and easier to use, and there's no reason to require the user to split the variable up, so convert the macros to use the combined argument.
--- kernel/sched/core.c | 22 +++++++++++----------- kernel/sched/debug.c | 4 ++-- kernel/sched/fair.c | 42 +++++++++++++++++++++--------------------- kernel/sched/idle_task.c | 2 +- kernel/sched/stats.h | 22 +++++++++++----------- 5 files changed, 46 insertions(+), 46 deletions(-)
kernel/sched/core.c +11 -11
@@ -1633,16 +1633,16 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags)int this_cpu =smp_processor_id ();if (cpu == this_cpu) { -schedstat_inc (rq, ttwu_local); -schedstat_inc (p, se.statistics.nr_wakeups_local); +schedstat_inc (rq->ttwu_local); +schedstat_inc (p->se.statistics.nr_wakeups_local); }else {struct sched_domain *sd; -schedstat_inc (p, se.statistics.nr_wakeups_remote); +schedstat_inc (p->se.statistics.nr_wakeups_remote);rcu_read_lock ();for_each_domain (this_cpu, sd) {if (cpumask_test_cpu (cpu,sched_domain_span (sd))) { -schedstat_inc (sd, ttwu_wake_remote); +schedstat_inc (sd->ttwu_wake_remote);break ; } }
@@ -1650,15 +1650,15 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags) }if (wake_flags &WF_MIGRATED ) -schedstat_inc (p, se.statistics.nr_wakeups_migrate); +schedstat_inc (p->se.statistics.nr_wakeups_migrate);#endif /* CONFIG_SMP */ -schedstat_inc (rq, ttwu_count); -schedstat_inc (p, se.statistics.nr_wakeups); +schedstat_inc (rq->ttwu_count); +schedstat_inc (p->se.statistics.nr_wakeups);if (wake_flags &WF_SYNC ) -schedstat_inc (p, se.statistics.nr_wakeups_sync); +schedstat_inc (p->se.statistics.nr_wakeups_sync);#endif /* CONFIG_SCHEDSTATS */ }
@@ -3181,7 +3181,7 @@ static inline void schedule_debug(struct task_struct *prev)profile_hit (SCHED_PROFILING ,__builtin_return_address (0 )); -schedstat_inc (this_rq (), sched_count); +schedstat_inc (this_rq ()->sched_count); }/*
@@ -4792,7 +4792,7 @@ SYSCALL_DEFINE0(sched_yield) {struct rq *rq =this_rq_lock (); -schedstat_inc (rq, yld_count); +schedstat_inc (rq->yld_count); current->sched_class->yield_task (rq);/*
@@ -4943,7 +4943,7 @@ again: yielded = curr->sched_class->yield_to_task (rq, p, preempt);if (yielded) { -schedstat_inc (rq, yld_count); +schedstat_inc (rq->yld_count);/* * Make p's CPU reschedule; pick_next_entity takes care of * fairness.
kernel/sched/debug.c +2 -2
@@ -429,9 +429,9 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) p->prio);SEQ_printf (m,"%9Ld.%06ld %9Ld.%06ld %9Ld.%06ld" , -SPLIT_NS (schedstat_val (p, se.statistics.wait_sum)), +SPLIT_NS (schedstat_val (p->se.statistics.wait_sum)),SPLIT_NS (p->se.sum_exec_runtime), -SPLIT_NS (schedstat_val (p, se.statistics.sum_sleep_runtime))); +SPLIT_NS (schedstat_val (p->se.statistics.sum_sleep_runtime)));#ifdef CONFIG_NUMA_BALANCING SEQ_printf (m," %d %d" ,task_node (p),task_numa_group_id (p));
kernel/sched/fair.c +21 -21
144 lines not on this page · raw message
kernel/sched/idle_task.c +1 -1
@@ -28,7 +28,7 @@ pick_next_task_idle(struct rq *rq, struct task_struct *prev, struct pin_cookie c {put_prev_task (rq, prev); -schedstat_inc (rq, sched_goidle); +schedstat_inc (rq->sched_goidle);return rq->idle; }
kernel/sched/stats.h +11 -11
@@ -29,11 +29,11 @@ rq_sched_info_dequeued(struct rq *rq, unsigned long long delta)if (rq) rq->rq_sched_info.run_delay += delta; } -# define schedstat_enabled () static_branch_unlikely(&sched_schedstats) -# define schedstat_inc (rq, field) do { if (schedstat_enabled()) { (rq)->field++; } } while (0) -# define schedstat_add (rq, field, amt) do { if (schedstat_enabled()) { (rq)->field += (amt); } } while (0) -# define schedstat_set (var, val) do { if (schedstat_enabled()) { var = (val); } } while (0) -# define schedstat_val (rq, field) ((schedstat_enabled()) ? (rq)->field : 0) +#define schedstat_enabled () static_branch_unlikely(&sched_schedstats) +#define schedstat_inc (var) do { if (schedstat_enabled()) { var++; } } while (0) +#define schedstat_add (var, amt) do { if (schedstat_enabled()) { var += (amt); } } while (0) +#define schedstat_set (var, val) do { if (schedstat_enabled()) { var = (val); } } while (0) +#define schedstat_val (var) ((schedstat_enabled()) ? (var) : 0)#else /* !CONFIG_SCHEDSTATS */ static inline void
@@ -45,12 +45,12 @@ rq_sched_info_dequeued(struct rq *rq, unsigned long long delta)static inline void rq_sched_info_depart (struct rq *rq,unsigned long long delta) {} -# define schedstat_enabled () 0 -# define schedstat_inc (rq, field) do { } while (0) -# define schedstat_add (rq, field, amt) do { } while (0) -# define schedstat_set (var, val) do { } while (0) -# define schedstat_val (rq, field) 0 -#endif +#define schedstat_enabled () 0 +#define schedstat_inc (var) do { } while (0) +#define schedstat_add (var, amt) do { } while (0) +#define schedstat_set (var, val) do { } while (0) +#define schedstat_val (var) 0 +#endif /* CONFIG_SCHEDSTATS */ #ifdef CONFIG_SCHED_INFO static inline void sched_info_reset_dequeued (struct task_struct *t)
Signature · 1 line
2.4.11
Quoted 15 lines · Josh Poimboeuf
NOTE: I didn't include any performance numbers because I wasn't able to get consistent results. I tried the following on a Xeon E5-2420 v2 CPU:
$ for i in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo -n performance > $i; done $ echo 1 > /sys/devices/system/cpu/intel_pstate/no_turbo $ echo 100 > /sys/devices/system/cpu/intel_pstate/min_perf_pct $ echo 0 > /proc/sys/kernel/nmi_watchdog $ taskset 0x10 perf stat -n -r10 perf bench sched pipe -l 1000000I was going to post the numbers from that, both with and without SCHEDSTATS, but then when I tried to repeat the test on a different day, the results were surprisingly different, with different conclusions.
So any advice on measuring scheduler performance would be appreciated...
Quoted 8 lines
Josh Poimboeuf (5):
sched/debug: rename and moveenqueue_sleeper()sched/debug: schedstat macro cleanup
sched/debug: 'schedstat_val()' -> 'schedstat_val_or_zero()' sched/debug: remove severalCONFIG_SCHEDSTATSguards sched/debug: decouple 'sched_stat_*' tracepoints' from
CONFIG_SCHEDSTATS
This patchset looks good to me.
Signature · 2 lines
Thanks and Regards Srikar Dronamraju
Peter Zijlstra
maintainerRe: [PATCH 4/5] sched/debug: remove several CONFIG_SCHEDSTATS guards
In reply to Josh PoimboeufQuoted 22 lines · Josh Poimboeuf
index 017a4f5..0aee5dd 100644
kernel/sched/core.c +7
-2@@ -1626,10 +1626,15 @@ static inline int __set_cpus_allowed_ptr(struct task_struct *p,static void ttwu_stat (struct task_struct *p,int cpu,int wake_flags) { -#ifdef CONFIG_SCHEDSTATS - struct rq *rq = this_rq(); +struct rq *rq; + +if (!schedstat_enabled ()) +return ; + + rq =this_rq ();#ifdef CONFIG_SMP +{int this_cpu =smp_processor_id ();if (cpu == this_cpu) {
kernel/sched/core.c ttwu_stat(struct task_struct *p, int cpu, int wake_flags) · line 1662
Quoted 8 lines
kernel/sched/core.c +1
-1@@ -1651,7 +1656,7 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags)if (wake_flags &WF_MIGRATED ) schedstat_inc(p->se.statistics.nr_wakeups_migrate); - +}#endif /* CONFIG_SMP */ schedstat_inc (rq->ttwu_count);
I did:
kernel/sched/core.c +2 -6
@@ -1636,10 +1636,7 @@ ttwu_stat(struct task_struct *p, int cpu rq =this_rq ();#ifdef CONFIG_SMP -{ -int this_cpu =smp_processor_id (); - -if (cpu == this_cpu) { +if (cpu == rq->cpu) {schedstat_inc (rq->ttwu_local);schedstat_inc (p->se.statistics.nr_wakeups_local); }else {
@@ -1647,7 +1644,7 @@ ttwu_stat(struct task_struct *p, int cpuschedstat_inc (p->se.statistics.nr_wakeups_remote);rcu_read_lock (); -for_each_domain (this_cpu, sd) { +for_each_domain (rq->cpu, sd) {if (cpumask_test_cpu (cpu,sched_domain_span (sd))) {schedstat_inc (sd->ttwu_wake_remote);break ;
@@ -1658,7 +1655,6 @@ ttwu_stat(struct task_struct *p, int cpuif (wake_flags &WF_MIGRATED )schedstat_inc (p->se.statistics.nr_wakeups_migrate); -}#endif /* CONFIG_SMP */ schedstat_inc (rq->ttwu_count);
Re: [PATCH 4/5] sched/debug: remove several CONFIG_SCHEDSTATS guards
Quoted 58 lines · Peter Zijlstra
Quoted 22 lines · Josh Poimboeuf
index 017a4f5..0aee5dd 100644
kernel/sched/core.c +7
-2@@ -1626,10 +1626,15 @@ static inline int __set_cpus_allowed_ptr(struct task_struct *p,static void ttwu_stat (struct task_struct *p,int cpu,int wake_flags) { -#ifdef CONFIG_SCHEDSTATS - struct rq *rq = this_rq(); +struct rq *rq; + +if (!schedstat_enabled ()) +return ; + + rq =this_rq ();#ifdef CONFIG_SMP +{int this_cpu =smp_processor_id ();if (cpu == this_cpu) {kernel/sched/core.c ttwu_stat(struct task_struct *p, int cpu, int wake_flags) · line 1662
Quoted 8 lines
kernel/sched/core.c +1
-1@@ -1651,7 +1656,7 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags)if (wake_flags &WF_MIGRATED ) schedstat_inc(p->se.statistics.nr_wakeups_migrate); - +}#endif /* CONFIG_SMP */ schedstat_inc (rq->ttwu_count);I did:
kernel/sched/core.c +2
-5@@ -1636,10 +1636,7 @@ ttwu_stat(struct task_struct *p, int cpu rq =this_rq ();#ifdef CONFIG_SMP -{ -int this_cpu =smp_processor_id (); - -if (cpu == this_cpu) { +if (cpu == rq->cpu) {schedstat_inc (rq->ttwu_local);schedstat_inc (p->se.statistics.nr_wakeups_local); }else {@@ -1647,7 +1644,7 @@ ttwu_stat(struct task_struct *p, int cpuschedstat_inc (p->se.statistics.nr_wakeups_remote);rcu_read_lock (); -for_each_domain (this_cpu, sd) { +for_each_domain (rq->cpu, sd) {if (cpumask_test_cpu (cpu,sched_domain_span (sd))) {schedstat_inc (sd->ttwu_wake_remote);break ;
kernel/sched/core.c ttwu_stat(struct task_struct *p, int cpu · line 1660
Quoted 7 lines
kernel/sched/core.c +0
-1@@ -1658,7 +1655,6 @@ ttwu_stat(struct task_struct *p, int cpuif (wake_flags &WF_MIGRATED )schedstat_inc (p->se.statistics.nr_wakeups_migrate); -}#endif /* CONFIG_SMP */ schedstat_inc (rq->ttwu_count);
Much better, thanks.
Signature · 1 line
Josh
Peter Zijlstra
maintainerQuoted 9 lines · Josh Poimboeuf
NOTE: I didn't include any performance numbers because I wasn't able to get consistent results. I tried the following on a Xeon E5-2420 v2 CPU:
$ for i in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo -n performance > $i; done $ echo 1 > /sys/devices/system/cpu/intel_pstate/no_turbo $ echo 100 > /sys/devices/system/cpu/intel_pstate/min_perf_pct $ echo 0 > /proc/sys/kernel/nmi_watchdog $ taskset 0x10 perf stat -n -r10 perf bench sched pipe -l 1000000
Quoted 5 lines
I was going to post the numbers from that, both with and without SCHEDSTATS, but then when I tried to repeat the test on a different day, the results were surprisingly different, with different conclusions.
So any advice on measuring scheduler performance would be appreciated...
Yeah, its a bit of a pain in general...
A) perf stat --null --repeat 50 -- perf bench sched messaging -g 50 -l 5000 | grep "seconds time elapsed"
B) perf stat --null --repeat 50 -- taskset 1 perf bench sched pipe | grep "seconds time elapsed"
1) tip/master + 1-4
2) tip/master + 1-5
3) tip/master + 1-5 + below
1 2 3
A) 4.627767855 4.650429917 4.646208062
4.633921933 4.641424424 4.612021058 4.649536375 4.663144144 4.636815948 4.630165619 4.649053552 4.613022902
B) 1.770732957 1.789534273 1.773334291
1.761740716 1.795618428 1.773338681 1.763761666 1.822316496 1.774385589
Quoted 1 line
From this it looks like patch 5 does hurt a wee bit, but we can get most
of that back by reordering the structure a bit. The results seem 'stable' across rebuilds and reboots (I've pop'ed all patches and rebuild, rebooted and re-benched 1 at the end and obtained similar results).
Although, possible that if we reorder first and then do 5, we'll just see a bigger regression. I've not bothered.
--- include/linux/sched.h | 33 +++++++++++++++------------------ kernel/sched/core.c | 4 ++-- kernel/sched/debug.c | 6 +++--- 3 files changed, 20 insertions(+), 23 deletions(-)
include/linux/sched.h +15 -18
@@ -1220,7 +1220,7 @@ struct uts_namespace;struct load_weight {unsigned long weight;u32 inv_weight; -}; +} __packed;/* * The load_avg/util_avg accumulates an infinite geometric series
@@ -1315,44 +1315,40 @@ struct sched_statistics {struct sched_entity {struct load_weight load;/* for load-balancing */ +unsigned int on_rq;struct rb_node run_node;struct list_head group_node; -unsigned int on_rq; -u64 exec_start; +u64 exec_start____cacheline_aligned_in_smp ;u64 sum_exec_runtime;u64 vruntime;u64 prev_sum_exec_runtime; - -u64 nr_migrations; -u64 wait_start;u64 sleep_start;u64 block_start; +#ifdef CONFIG_SMP +/* +* Per entity load average tracking. +*/ +struct sched_avg avg____cacheline_aligned_in_smp ; +#endif #ifdef CONFIG_SCHEDSTATS struct sched_statistics statistics;#endif #ifdef CONFIG_FAIR_GROUP_SCHED -int depth; +/* +* mostly constant values, separate from modifications above +*/ +int depth____cacheline_aligned_in_smp ;struct sched_entity *parent;/* rq on which this entity is (to be) queued: */ struct cfs_rq *cfs_rq;/* rq "owned" by this entity/group: */ struct cfs_rq *my_q;#endif - -#ifdef CONFIG_SMP -/* -* Per entity load average tracking. -* -* Put into separate cache line so it does not -* collide with read-mostly values above. -*/ -struct sched_avg avg____cacheline_aligned_in_smp ; -#endif -}; +}____cacheline_aligned_in_smp ;struct sched_rt_entity {struct list_head run_list;
@@ -1475,6 +1471,7 @@ struct task_struct {int prio, static_prio, normal_prio;unsigned int rt_priority;const struct sched_class *sched_class; +u64 nr_migrations;struct sched_entity se;struct sched_rt_entity rt;#ifdef CONFIG_CGROUP_SCHED
kernel/sched/core.c +2 -2
@@ -1239,7 +1239,7 @@ void set_task_cpu(struct task_struct *p,if (task_cpu (p) != new_cpu) {if (p->sched_class->migrate_task_rq) p->sched_class->migrate_task_rq (p); - p->se.nr_migrations++; + p->nr_migrations++;perf_event_task_migrate (p); }
@@ -2167,7 +2167,7 @@ static void __sched_fork(unsigned long c p->se.exec_start =0 ; p->se.sum_exec_runtime =0 ; p->se.prev_sum_exec_runtime =0 ; - p->se.nr_migrations =0 ; + p->nr_migrations =0 ; p->se.vruntime =0 ;INIT_LIST_HEAD (&p->se.group_node);
kernel/sched/debug.c +3 -3
@@ -885,7 +885,7 @@ void proc_sched_show_task(struct task_st nr_switches = p->nvcsw + p->nivcsw; -P (se.nr_migrations); +P (nr_migrations);PN (se.wait_start);PN (se.sleep_start);
@@ -926,9 +926,9 @@ void proc_sched_show_task(struct task_st avg_atom =-1LL ; avg_per_cpu = p->se.sum_exec_runtime; -if (p->se.nr_migrations) { +if (p->nr_migrations) { avg_per_cpu =div64_u64 (avg_per_cpu, - p->se.nr_migrations); + p->nr_migrations); }else { avg_per_cpu =-1LL ; }
Quoted 37 lines · Peter Zijlstra
Quoted 9 lines · Josh Poimboeuf
NOTE: I didn't include any performance numbers because I wasn't able to get consistent results. I tried the following on a Xeon E5-2420 v2 CPU:
$ for i in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo -n performance > $i; done $ echo 1 > /sys/devices/system/cpu/intel_pstate/no_turbo $ echo 100 > /sys/devices/system/cpu/intel_pstate/min_perf_pct $ echo 0 > /proc/sys/kernel/nmi_watchdog $ taskset 0x10 perf stat -n -r10 perf bench sched pipe -l 1000000Quoted 5 lines
I was going to post the numbers from that, both with and without SCHEDSTATS, but then when I tried to repeat the test on a different day, the results were surprisingly different, with different conclusions.
So any advice on measuring scheduler performance would be appreciated...
Yeah, its a bit of a pain in general...
A) perf stat --null --repeat 50 -- perf bench sched messaging -g 50 -l 5000 | grep "seconds time elapsed"
B) perf stat --null --repeat 50 -- taskset 1 perf bench sched pipe | grep "seconds time elapsed"1) tip/master + 1-4
2) tip/master + 1-5
3) tip/master + 1-5 + below1 2 3A) 4.627767855 4.650429917 4.646208062
4.633921933 4.641424424 4.612021058 4.649536375 4.663144144 4.636815948 4.630165619 4.649053552 4.613022902B) 1.770732957 1.789534273 1.773334291
1.761740716 1.795618428 1.773338681 1.763761666 1.822316496 1.774385589
Quoted 8 lines
From this it looks like patch 5 does hurt a wee bit, but we can get most of that back by reordering the structure a bit. The results seem 'stable' across rebuilds and reboots (I've pop'ed all patches and rebuild, rebooted and re-benched 1 at the end and obtained similar results).
Although, possible that if we reorder first and then do 5, we'll just see a bigger regression. I've not bothered.
Thanks a lot for benchmarking this! And also for improving the cache alignments. Your changes look good to me.
Signature · 1 line
Josh
Peter Zijlstra
maintainerQuoted 21 lines · Peter Zijlstra
Yeah, its a bit of a pain in general...
A) perf stat --null --repeat 50 -- perf bench sched messaging -g 50 -l 5000 | grep "seconds time elapsed"
B) perf stat --null --repeat 50 -- taskset 1 perf bench sched pipe | grep "seconds time elapsed"1) tip/master + 1-4
2) tip/master + 1-5
3) tip/master + 1-5 + below1 2 3A) 4.627767855 4.650429917 4.646208062
4.633921933 4.641424424 4.612021058 4.649536375 4.663144144 4.636815948 4.630165619 4.649053552 4.613022902B) 1.770732957 1.789534273 1.773334291
1.761740716 1.795618428 1.773338681 1.763761666 1.822316496 1.774385589
Quoted 5 lines
From this it looks like patch 5 does hurt a wee bit, but we can get most of that back by reordering the structure a bit. The results seem 'stable' across rebuilds and reboots (I've pop'ed all patches and rebuild, rebooted and re-benched 1 at the end and obtained similar results).
Ha! So those numbers were with CONFIG_SCHEDSTAT=n :-/
1) above 1 (4 patches, CONFIG_SCHEDSTAT=n, sysctl=0)
2) 1 + CONFIG_SCHEDSTAT=y (sysctl=0)
3) 2 + sysctl=1
4) above 3 (6 patches) + CONFIG_SCHEDSTAT=y (sysctl=0)
1 2 3 4
A) 4.620495664 4.788352823 4.862036428 4.623480512
4.628800053 4.792622881 4.855325525 4.613553872 4.611909507 4.794282178 4.850959761 4.613323142 4.608379522 4.787300153 4.822439864 4.597903070
B) 1.765668026 1.788374847 1.877803100 1.827213170
1.769379968 1.779881911 1.870091005 1.825335322 1.765822150 1.786251610 1.885874745 1.828218761
Which looks good for hackbench, but still stinks for pipetest :/
Quoted 44 lines · Peter Zijlstra
Quoted 21 lines · Peter Zijlstra
Yeah, its a bit of a pain in general...
A) perf stat --null --repeat 50 -- perf bench sched messaging -g 50 -l 5000 | grep "seconds time elapsed"
B) perf stat --null --repeat 50 -- taskset 1 perf bench sched pipe | grep "seconds time elapsed"1) tip/master + 1-4
2) tip/master + 1-5
3) tip/master + 1-5 + below1 2 3A) 4.627767855 4.650429917 4.646208062
4.633921933 4.641424424 4.612021058 4.649536375 4.663144144 4.636815948 4.630165619 4.649053552 4.613022902B) 1.770732957 1.789534273 1.773334291
1.761740716 1.795618428 1.773338681 1.763761666 1.822316496 1.774385589Quoted 5 lines
From this it looks like patch 5 does hurt a wee bit, but we can get most of that back by reordering the structure a bit. The results seem 'stable' across rebuilds and reboots (I've pop'ed all patches and rebuild, rebooted and re-benched 1 at the end and obtained similar results).
Ha! So those numbers were with
CONFIG_SCHEDSTAT=n:-/1) above 1 (4 patches,
CONFIG_SCHEDSTAT=n, sysctl=0)
2) 1 +CONFIG_SCHEDSTAT=y(sysctl=0)
3) 2 + sysctl=1
4) above 3 (6 patches) +CONFIG_SCHEDSTAT=y(sysctl=0)1 2 3 4A) 4.620495664 4.788352823 4.862036428 4.623480512
4.628800053 4.792622881 4.855325525 4.613553872 4.611909507 4.794282178 4.850959761 4.613323142 4.608379522 4.787300153 4.822439864 4.597903070
Quoted 6 lines
B) 1.765668026 1.788374847 1.877803100 1.827213170
1.769379968 1.779881911 1.870091005 1.825335322 1.765822150 1.786251610 1.885874745 1.828218761Which looks good for hackbench, but still stinks for pipetest :/
I tried again on another system (Broadwell 2*10*2) and seemingly got more consistent results, but the conclusions are a bit different from yours.
I tested only with CONFIG_SCHEDSTAT=y, sysctl=0, because I think that should be the most common configuration by far.
1) linus/master
2) linus/master + 1-4
3) linux/master + 1-5
4) linus/master + 1-5 + smp cacheline patch
A) perf stat --null --repeat 50 -- perf bench sched messaging -g 50 -l 5000
B) perf stat --null --repeat 50 -- taskset 1 perf bench sched pipe
1 2 3 4
A) 6.335625627 6.299825679 6.317633969 6.305548464
6.334188492 6.331391159 6.345195048 6.334608006 6.345243359 6.329650737 6.328263309 6.304355127 6.333154970 6.313859694 6.336338820 6.342374680
B) 2.310476138 2.324716175 2.355990033 2.350834083
2.307231831 2.327946052 2.349816680 2.335581939 2.303859470 2.317300965 2.347193526 2.333758084 2.317224538 2.331390610 2.326164933 2.334235895
With patches 1-4, hackbench was slightly better and pipetest was slightly worse.
With patches 1-5, hackbench was about the same or even slightly better than baseline, and pipetest was 1-2% worse than baseline.
With your smp cacheline patch added, I didn't see a clear improvement.
It would be nice to have the schedstat tracepoints be always functional, but I suppose it's up to you and Ingo as to whether it's worth the performance tradeoff.
Another option would be to only merge patches 1-4.
Signature · 1 line
Josh
tip-bot for Josh Poimboeuf
merge bot[tip:sched/core] sched/debug: Rename and move enqueue_sleeper()
In reply to Josh Poimboeufsched/debug: Rename and move enqueue_sleeper()
Commit-ID: 1a3d027c5a6847e5d349c8527f99aada47e5467a Gitweb: http://git.kernel.org/tip/1a3d027c5a6847e5d349c8527f99aada47e5467a Author: Josh Poimboeuf <[email protected]> AuthorDate: Fri, 17 Jun 2016 12:43:23 -0500 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 5 Sep 2016 13:29:45 +0200
sched/debug: Rename and move enqueue_sleeper()
enqueue_sleeper() doesn't actually enqueue, it just handles some statistics and tracepoints. Rename it to update_stats_enqueue_sleeper() and call it from update_stats_enqueue().
--- kernel/sched/fair.c | 142 +++++++++++++++++++++++++++------------------------- 1 file changed, 73 insertions(+), 69 deletions(-)
kernel/sched/fair.c +73 -69
@@ -862,11 +862,72 @@ update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se) se->statistics.wait_start =0 ; } +static void +update_stats_enqueue_sleeper (struct cfs_rq *cfs_rq,struct sched_entity *se) +{ +struct task_struct *tsk =NULL ; + +if (entity_is_task (se)) + tsk =task_of (se); + +if (se->statistics.sleep_start) { +u64 delta =rq_clock (rq_of (cfs_rq)) - se->statistics.sleep_start; + +if ((s64 )delta <0 ) + delta =0 ; + +if (unlikely (delta > se->statistics.sleep_max)) + se->statistics.sleep_max = delta; + + se->statistics.sleep_start =0 ; + se->statistics.sum_sleep_runtime += delta; + +if (tsk) { +account_scheduler_latency (tsk, delta >>10 ,1 ); +trace_sched_stat_sleep (tsk, delta); + } + } +if (se->statistics.block_start) { +u64 delta =rq_clock (rq_of (cfs_rq)) - se->statistics.block_start; + +if ((s64 )delta <0 ) + delta =0 ; + +if (unlikely (delta > se->statistics.block_max)) + se->statistics.block_max = delta; + + se->statistics.block_start =0 ; + se->statistics.sum_sleep_runtime += delta; + +if (tsk) { +if (tsk->in_iowait) { + se->statistics.iowait_sum += delta; + se->statistics.iowait_count++; +trace_sched_stat_iowait (tsk, delta); + } + +trace_sched_stat_blocked (tsk, delta); + +/* +* Blocking time is in units of nanosecs, so shift by +* 20 to get a milliseconds-range estimation of the +* amount of time that the task spent sleeping: +*/ +if (unlikely (prof_on ==SLEEP_PROFILING )) { +profile_hits (SLEEP_PROFILING , + (void *)get_wchan (tsk), + delta >>20 ); + } +account_scheduler_latency (tsk, delta >>10 ,0 ); + } + } +} +/* * Task is being enqueued - update stats: */ static inline void -update_stats_enqueue (struct cfs_rq *cfs_rq,struct sched_entity *se) +update_stats_enqueue (struct cfs_rq *cfs_rq,struct sched_entity *se,int flags) {/* * Are we enqueueing a waiting task? (for current tasks
@@ -874,6 +935,9 @@ update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)*/ if (se != cfs_rq->curr)update_stats_wait_start (cfs_rq, se); + +if (flags &ENQUEUE_WAKEUP ) +update_stats_enqueue_sleeper (cfs_rq, se); } static inline void
@@ -910,7 +974,12 @@ update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se) }static inline void -update_stats_enqueue (struct cfs_rq *cfs_rq,struct sched_entity *se) +update_stats_enqueue_sleeper (struct cfs_rq *cfs_rq,struct sched_entity *se) +{ +} + +static inline void +update_stats_enqueue (struct cfs_rq *cfs_rq,struct sched_entity *se,int flags) { }
@@ -3197,68 +3266,6 @@ static inline int idle_balance(struct rq *rq)#endif /* CONFIG_SMP */ -static void enqueue_sleeper (struct cfs_rq *cfs_rq,struct sched_entity *se) -{ -#ifdef CONFIG_SCHEDSTATS -struct task_struct *tsk =NULL ; - -if (entity_is_task (se)) - tsk =task_of (se); - -if (se->statistics.sleep_start) { -u64 delta =rq_clock (rq_of (cfs_rq)) - se->statistics.sleep_start; - -if ((s64 )delta <0 ) - delta =0 ; - -if (unlikely (delta > se->statistics.sleep_max)) - se->statistics.sleep_max = delta; - - se->statistics.sleep_start =0 ; - se->statistics.sum_sleep_runtime += delta; - -if (tsk) { -account_scheduler_latency (tsk, delta >>10 ,1 ); -trace_sched_stat_sleep (tsk, delta); - } - } -if (se->statistics.block_start) { -u64 delta =rq_clock (rq_of (cfs_rq)) - se->statistics.block_start; - -if ((s64 )delta <0 ) - delta =0 ; - -if (unlikely (delta > se->statistics.block_max)) - se->statistics.block_max = delta; - - se->statistics.block_start =0 ; - se->statistics.sum_sleep_runtime += delta; - -if (tsk) { -if (tsk->in_iowait) { - se->statistics.iowait_sum += delta; - se->statistics.iowait_count++; -trace_sched_stat_iowait (tsk, delta); - } - -trace_sched_stat_blocked (tsk, delta); - -/* -* Blocking time is in units of nanosecs, so shift by -* 20 to get a milliseconds-range estimation of the -* amount of time that the task spent sleeping: -*/ -if (unlikely (prof_on ==SLEEP_PROFILING )) { -profile_hits (SLEEP_PROFILING , - (void *)get_wchan (tsk), - delta >>20 ); - } -account_scheduler_latency (tsk, delta >>10 ,0 ); - } - } -#endif -} -static void check_spread (struct cfs_rq *cfs_rq,struct sched_entity *se) {#ifdef CONFIG_SCHED_DEBUG
@@ -3385,15 +3392,12 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)account_entity_enqueue (cfs_rq, se);update_cfs_shares (cfs_rq); -if (flags &ENQUEUE_WAKEUP ) { +if (flags &ENQUEUE_WAKEUP )place_entity (cfs_rq, se,0 ); -if (schedstat_enabled ()) -enqueue_sleeper (cfs_rq, se); - }check_schedstat_required ();if (schedstat_enabled ()) { -update_stats_enqueue (cfs_rq, se); +update_stats_enqueue (cfs_rq, se, flags);check_spread (cfs_rq, se); }if (!curr)
tip-bot for Josh Poimboeuf
merge bot[tip:sched/core] sched/debug: Rename 'schedstat_val()' -> 'schedstat_val_or_zero()'
In reply to Josh Poimboeufsched/debug: Rename 'schedstat_val()' -> 'schedstat_val_or_zero()'
Commit-ID: 20e1d4863bfa7152e98f94e5bcdda3e7db41d899 Gitweb: http://git.kernel.org/tip/20e1d4863bfa7152e98f94e5bcdda3e7db41d899 Author: Josh Poimboeuf <[email protected]> AuthorDate: Fri, 17 Jun 2016 12:43:25 -0500 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 5 Sep 2016 13:29:46 +0200
sched/debug: Rename 'schedstat_val()' -> 'schedstat_val_or_zero()'
The schedstat_val() macro's behavior is kind of surprising: when schedstat is runtime disabled, it returns zero. Rename it to schedstat_val_or_zero().
There's also a need for a similar macro which doesn't have the 'if (schedstat_enable())' check, to avoid doing the check twice. Create a new 'schedstat_val()' macro for that.
--- kernel/sched/debug.c | 4 ++-- kernel/sched/stats.h | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-)
kernel/sched/debug.c +2 -2
@@ -429,9 +429,9 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) p->prio);SEQ_printf (m,"%9Ld.%06ld %9Ld.%06ld %9Ld.%06ld" , -SPLIT_NS (schedstat_val (p->se.statistics.wait_sum)), +SPLIT_NS (schedstat_val_or_zero (p->se.statistics.wait_sum)),SPLIT_NS (p->se.sum_exec_runtime), -SPLIT_NS (schedstat_val (p->se.statistics.sum_sleep_runtime))); +SPLIT_NS (schedstat_val_or_zero (p->se.statistics.sum_sleep_runtime)));#ifdef CONFIG_NUMA_BALANCING SEQ_printf (m," %d %d" ,task_node (p),task_numa_group_id (p));
kernel/sched/stats.h +3 -1
@@ -33,7 +33,8 @@ rq_sched_info_dequeued(struct rq *rq, unsigned long long delta)#define schedstat_inc (var) do { if (schedstat_enabled()) { var++; } } while (0)#define schedstat_add (var, amt) do { if (schedstat_enabled()) { var += (amt); } } while (0)#define schedstat_set (var, val) do { if (schedstat_enabled()) { var = (val); } } while (0) -#define schedstat_val (var) ((schedstat_enabled()) ? (var) : 0) +#define schedstat_val (var) (var) +#define schedstat_val_or_zero (var) ((schedstat_enabled()) ? (var) : 0)#else /* !CONFIG_SCHEDSTATS */ static inline void
@@ -50,6 +51,7 @@ rq_sched_info_depart(struct rq *rq, unsigned long long delta)#define schedstat_add (var, amt) do { } while (0)#define schedstat_set (var, val) do { } while (0)#define schedstat_val (var) 0 +#define schedstat_val_or_zero (var) 0#endif /* CONFIG_SCHEDSTATS */ #ifdef CONFIG_SCHED_INFO
tip-bot for Josh Poimboeuf
merge bot[tip:sched/core] sched/debug: Remove several CONFIG_SCHEDSTATS guards
In reply to Josh Poimboeufsched/debug: Remove several CONFIG_SCHEDSTATS guards
Commit-ID: 4fa8d299b43a91f871f6d5b00dd5ab33d43bbc2c Gitweb: http://git.kernel.org/tip/4fa8d299b43a91f871f6d5b00dd5ab33d43bbc2c Author: Josh Poimboeuf <[email protected]> AuthorDate: Fri, 17 Jun 2016 12:43:26 -0500 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 5 Sep 2016 13:29:47 +0200
sched/debug: Remove several CONFIG_SCHEDSTATS guards
Clean up the sched code by removing several of the CONFIG_SCHEDSTATS guards, using schedstat_*() macros where needed.
Code size:
!CONFIG_SCHEDSTATS defconfig:
text data bss dec hex filename
10209818 4368184 1105920 15683922 ef5152 vmlinux.before.nostats
10209818 4368184 1105920 15683922 ef5152 vmlinux.after.nostats
CONFIG_SCHEDSTATS defconfig:
text data bss dec hex filename
10214210 4370040 1105920 15690170 ef69ba vmlinux.before.stats
10214210 4370680 1105920 15690810 ef6c3a vmlinux.after.stats
--- kernel/sched/core.c | 33 +++++------- kernel/sched/debug.c | 99 ++++++++++++++++++---------------- kernel/sched/fair.c | 148 ++++++++++++++++++++++++--------------------------- 3 files changed, 136 insertions(+), 144 deletions(-)
kernel/sched/core.c +14 -19
@@ -1629,13 +1629,15 @@ static inline int __set_cpus_allowed_ptr(struct task_struct *p,static void ttwu_stat (struct task_struct *p,int cpu,int wake_flags) { -#ifdef CONFIG_SCHEDSTATS - struct rq *rq = this_rq(); +struct rq *rq; -#ifdef CONFIG_SMP - int this_cpu = smp_processor_id(); +if (!schedstat_enabled ()) +return ; + + rq =this_rq (); - if (cpu == this_cpu) { +#ifdef CONFIG_SMP +if (cpu == rq->cpu) {schedstat_inc (rq->ttwu_local);schedstat_inc (p->se.statistics.nr_wakeups_local); }else {
@@ -1643,7 +1645,7 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags)schedstat_inc (p->se.statistics.nr_wakeups_remote);rcu_read_lock (); -for_each_domain (this_cpu, sd) { +for_each_domain (rq->cpu, sd) {if (cpumask_test_cpu (cpu,sched_domain_span (sd))) {schedstat_inc (sd->ttwu_wake_remote);break ;
@@ -1654,7 +1656,6 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags)if (wake_flags &WF_MIGRATED )schedstat_inc (p->se.statistics.nr_wakeups_migrate); -#endif /* CONFIG_SMP */ schedstat_inc (rq->ttwu_count);
@@ -1662,8 +1663,6 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags)if (wake_flags &WF_SYNC )schedstat_inc (p->se.statistics.nr_wakeups_sync); - -#endif /* CONFIG_SCHEDSTATS */ }static inline void ttwu_activate (struct rq *rq,struct task_struct *p,int en_flags)
@@ -2084,8 +2083,7 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)ttwu_queue (p, cpu, wake_flags); stat: -if (schedstat_enabled ()) -ttwu_stat (p, cpu, wake_flags); +ttwu_stat (p, cpu, wake_flags); out:raw_spin_unlock_irqrestore (&p->pi_lock, flags);
@@ -2134,8 +2132,7 @@ static void try_to_wake_up_local(struct task_struct *p, struct pin_cookie cookiettwu_activate (rq, p,ENQUEUE_WAKEUP );ttwu_do_wakeup (rq, p,0 , cookie); -if (schedstat_enabled ()) -ttwu_stat (p,smp_processor_id (),0 ); +ttwu_stat (p,smp_processor_id (),0 ); out:raw_spin_unlock (&p->pi_lock); }
@@ -7675,12 +7672,10 @@ void normalize_rt_tasks(void)if (p->flags &PF_KTHREAD )continue ; - p->se.exec_start =0 ; -#ifdef CONFIG_SCHEDSTATS - p->se.statistics.wait_start =0 ; - p->se.statistics.sleep_start =0 ; - p->se.statistics.block_start =0 ; -#endif + p->se.exec_start =0 ; +schedstat_set (p->se.statistics.wait_start,0 ); +schedstat_set (p->se.statistics.sleep_start,0 ); +schedstat_set (p->se.statistics.block_start,0 );if (!dl_task (p) && !rt_task (p)) {/*
kernel/sched/debug.c +53 -46
167 lines not on this page · raw message
kernel/sched/fair.c +69 -79
267 lines not on this page · raw message
tip-bot for Josh Poimboeuf
merge bot[tip:sched/core] sched/debug: Clean up schedstat macros
In reply to Josh Poimboeufsched/debug: Clean up schedstat macros
Commit-ID: ae92882e5646d8661a3ca182ba988752fe4b773f Gitweb: http://git.kernel.org/tip/ae92882e5646d8661a3ca182ba988752fe4b773f Author: Josh Poimboeuf <[email protected]> AuthorDate: Fri, 17 Jun 2016 12:43:24 -0500 Committer: Ingo Molnar <[email protected]> CommitDate: Mon, 5 Sep 2016 13:29:46 +0200
sched/debug: Clean up schedstat macros
The schedstat_*() macros are inconsistent: most of them take a pointer and a field which the macro combines, whereas schedstat_set() takes the already combined ptr->field.
The already combined ptr->field argument is actually more intuitive and easier to use, and there's no reason to require the user to split the variable up, so convert the macros to use the combined argument.
--- kernel/sched/core.c | 22 +++++++++++----------- kernel/sched/debug.c | 4 ++-- kernel/sched/fair.c | 42 +++++++++++++++++++++--------------------- kernel/sched/idle_task.c | 2 +- kernel/sched/stats.h | 22 +++++++++++----------- 5 files changed, 46 insertions(+), 46 deletions(-)
kernel/sched/core.c +11 -11
@@ -1636,16 +1636,16 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags)int this_cpu =smp_processor_id ();if (cpu == this_cpu) { -schedstat_inc (rq, ttwu_local); -schedstat_inc (p, se.statistics.nr_wakeups_local); +schedstat_inc (rq->ttwu_local); +schedstat_inc (p->se.statistics.nr_wakeups_local); }else {struct sched_domain *sd; -schedstat_inc (p, se.statistics.nr_wakeups_remote); +schedstat_inc (p->se.statistics.nr_wakeups_remote);rcu_read_lock ();for_each_domain (this_cpu, sd) {if (cpumask_test_cpu (cpu,sched_domain_span (sd))) { -schedstat_inc (sd, ttwu_wake_remote); +schedstat_inc (sd->ttwu_wake_remote);break ; } }
@@ -1653,15 +1653,15 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags) }if (wake_flags &WF_MIGRATED ) -schedstat_inc (p, se.statistics.nr_wakeups_migrate); +schedstat_inc (p->se.statistics.nr_wakeups_migrate);#endif /* CONFIG_SMP */ -schedstat_inc (rq, ttwu_count); -schedstat_inc (p, se.statistics.nr_wakeups); +schedstat_inc (rq->ttwu_count); +schedstat_inc (p->se.statistics.nr_wakeups);if (wake_flags &WF_SYNC ) -schedstat_inc (p, se.statistics.nr_wakeups_sync); +schedstat_inc (p->se.statistics.nr_wakeups_sync);#endif /* CONFIG_SCHEDSTATS */ }
@@ -3237,7 +3237,7 @@ static inline void schedule_debug(struct task_struct *prev)profile_hit (SCHED_PROFILING ,__builtin_return_address (0 )); -schedstat_inc (this_rq (), sched_count); +schedstat_inc (this_rq ()->sched_count); }/*
@@ -4849,7 +4849,7 @@ SYSCALL_DEFINE0(sched_yield) {struct rq *rq =this_rq_lock (); -schedstat_inc (rq, yld_count); +schedstat_inc (rq->yld_count); current->sched_class->yield_task (rq);/*
@@ -5000,7 +5000,7 @@ again: yielded = curr->sched_class->yield_to_task (rq, p, preempt);if (yielded) { -schedstat_inc (rq, yld_count); +schedstat_inc (rq->yld_count);/* * Make p's CPU reschedule; pick_next_entity takes care of * fairness.
kernel/sched/debug.c +2 -2
@@ -429,9 +429,9 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) p->prio);SEQ_printf (m,"%9Ld.%06ld %9Ld.%06ld %9Ld.%06ld" , -SPLIT_NS (schedstat_val (p, se.statistics.wait_sum)), +SPLIT_NS (schedstat_val (p->se.statistics.wait_sum)),SPLIT_NS (p->se.sum_exec_runtime), -SPLIT_NS (schedstat_val (p, se.statistics.sum_sleep_runtime))); +SPLIT_NS (schedstat_val (p->se.statistics.sum_sleep_runtime)));#ifdef CONFIG_NUMA_BALANCING SEQ_printf (m," %d %d" ,task_node (p),task_numa_group_id (p));
kernel/sched/fair.c +21 -21
144 lines not on this page · raw message
kernel/sched/idle_task.c +1 -1
@@ -28,7 +28,7 @@ pick_next_task_idle(struct rq *rq, struct task_struct *prev, struct pin_cookie c {put_prev_task (rq, prev); -schedstat_inc (rq, sched_goidle); +schedstat_inc (rq->sched_goidle);return rq->idle; }
kernel/sched/stats.h +11 -11
@@ -29,11 +29,11 @@ rq_sched_info_dequeued(struct rq *rq, unsigned long long delta)if (rq) rq->rq_sched_info.run_delay += delta; } -# define schedstat_enabled () static_branch_unlikely(&sched_schedstats) -# define schedstat_inc (rq, field) do { if (schedstat_enabled()) { (rq)->field++; } } while (0) -# define schedstat_add (rq, field, amt) do { if (schedstat_enabled()) { (rq)->field += (amt); } } while (0) -# define schedstat_set (var, val) do { if (schedstat_enabled()) { var = (val); } } while (0) -# define schedstat_val (rq, field) ((schedstat_enabled()) ? (rq)->field : 0) +#define schedstat_enabled () static_branch_unlikely(&sched_schedstats) +#define schedstat_inc (var) do { if (schedstat_enabled()) { var++; } } while (0) +#define schedstat_add (var, amt) do { if (schedstat_enabled()) { var += (amt); } } while (0) +#define schedstat_set (var, val) do { if (schedstat_enabled()) { var = (val); } } while (0) +#define schedstat_val (var) ((schedstat_enabled()) ? (var) : 0)#else /* !CONFIG_SCHEDSTATS */ static inline void
@@ -45,12 +45,12 @@ rq_sched_info_dequeued(struct rq *rq, unsigned long long delta)static inline void rq_sched_info_depart (struct rq *rq,unsigned long long delta) {} -# define schedstat_enabled () 0 -# define schedstat_inc (rq, field) do { } while (0) -# define schedstat_add (rq, field, amt) do { } while (0) -# define schedstat_set (var, val) do { } while (0) -# define schedstat_val (rq, field) 0 -#endif +#define schedstat_enabled () 0 +#define schedstat_inc (var) do { } while (0) +#define schedstat_add (var, amt) do { } while (0) +#define schedstat_set (var, val) do { } while (0) +#define schedstat_val (var) 0 +#endif /* CONFIG_SCHEDSTATS */ #ifdef CONFIG_SCHED_INFO static inline void sched_info_reset_dequeued (struct task_struct *t)