sched: Add cpu based entries to debugfs
RFC PATCH 1/1 sched: Add cpu based entries to debugfs
Currently sched_debug can be added to the kernel commandline parameters to dump domain information during boot. This method is not practical with a large number of CPUs.
This patch adds per-cpu entries to debugfs under a sched directory. Reading the per-cpu file shows the domain information in a human-readable format:
$ cat /sys/kernel/debug/sched/cpu0
domain 0 / SMT:
flags: 0x2af: load-balance new-idle exec fork affine cpu-capacity share-pkg-resources
span: 0-7
groups:
0 (cpu_capacity = 147)
1 (cpu_capacity = 147)
2 (cpu_capacity = 147)
3 (cpu_capacity = 147)
4 (cpu_capacity = 147)
5 (cpu_capacity = 147)
6 (cpu_capacity = 147)
7 (cpu_capacity = 147)domain 2 / DIE:
flags: 0x102f: load-balance new-idle exec fork affine prefer-sibling
span: 0-127
groups:
0-7 (cpu_capacity = 1176)
8-15 (cpu_capacity = 1176)
16-23 (cpu_capacity = 1176)
24-31 (cpu_capacity = 1176)
32-39 (cpu_capacity = 1176)
40-47 (cpu_capacity = 1176)
48-55 (cpu_capacity = 1176)
56-63 (cpu_capacity = 1176)
64-71 (cpu_capacity = 1176)
72-79 (cpu_capacity = 1176)
80-87 (cpu_capacity = 1176)
88-95 (cpu_capacity = 1176)
96-103 (cpu_capacity = 1176)
104-111 (cpu_capacity = 1176)
112-119 (cpu_capacity = 1176)
120-127 (cpu_capacity = 1176)domain 3 / NUMA:
flags: 0x642f: load-balance new-idle exec fork affine serialize overlap numa
span: 0-1023
groups:
0-127 (cpu_capacity = 18816)
128-255 (cpu_capacity = 18816)
256-383 (cpu_capacity = 18816)
384-511 (cpu_capacity = 18816)
512-639 (cpu_capacity = 18816)
640-767 (cpu_capacity = 18816)
768-895 (cpu_capacity = 18816)
896-1023 (cpu_capacity = 18816)Before spending too much time formalizing this I wanted to see if you guys would entertain the idea of making this info available via debugfs. It does move the existing sched_features file to sched/features -- not sure how acceptable it is to move files in debugfs.
TO-DO: handle hotplug
--- kernel/sched/core.c | 167 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 164 insertions(+), 3 deletions(-)
kernel/sched/core.c +164 -3
@@ -268,12 +268,173 @@ static const struct file_operations sched_feat_fops = { .release = single_release, }; +static const char *const sd_flag_names[] = { +"load-balance" , +"new-idle" , +"exec" , +"fork" , +"wake" , +"affine" , +"" , +"cpu-capacity" , +"power-domain" , +"share-pkg-resources" , +"serialize" , +"asym-packing" , +"prefer-sibling" , +"overlap" , +"numa" , +"" , +}; +static void sched_cpu_domain_show (struct seq_file *m,struct sched_domain *sd, +int cpu) +{ +struct cpumask groupmask; +struct sched_group *group = sd->groups; +int i; + +cpumask_clear (&groupmask); + +seq_printf (m,"domain %d / %s:\n" , sd->level, sd->name); +seq_printf (m," flags: 0x%x: " , sd->flags); + +for (i =0 ; i <ARRAY_SIZE (sd_flag_names); ++i) { +if (sd->flags & (1 << i)) +seq_printf (m," %s" , sd_flag_names[i]); + } +seq_puts (m,"\n" ); + +if (!(sd->flags &SD_LOAD_BALANCE ) && sd->parent) +seq_puts (m," ERROR: !SD_LOAD_BALANCE domain has parent\n" ); + +seq_printf (m," span: %*pbl\n" , +cpumask_pr_args (sched_domain_span (sd))); + +if (!cpumask_test_cpu (cpu,sched_domain_span (sd))) +seq_printf (m," ERROR: domain->span does not contain CPU%d\n" , cpu); + +if (!cpumask_test_cpu (cpu,sched_group_cpus (group))) +seq_printf (m," ERROR: domain->groups does not contain CPU%d\n" , cpu); + +seq_puts (m," groups:\n" ); +do { +if (!group) { +seq_puts (m," ERROR: group is NULL\n" ); +break ; + } + +/* +* Even though we initialize ->capacity to something semi-sane, +* we leave capacity_orig unset. This allows us to detect if +* domain iteration is still funny without causing /0 traps. +*/ +if (!group->sgc->capacity_orig) { +seq_puts (m," ERROR: domain->cpu_capacity not set\n" ); +break ; + } + +if (!cpumask_weight (sched_group_cpus (group))) { +seq_puts (m," ERROR: empty group\n" ); +break ; + } + +if (!(sd->flags &SD_OVERLAP ) && +cpumask_intersects (&groupmask,sched_group_cpus (group))) { +seq_puts (m," ERROR: repeated CPUs\n" ); +break ; + } + +cpumask_or (&groupmask, &groupmask,sched_group_cpus (group)); + +seq_printf (m," %*pbl" , +cpumask_pr_args (sched_group_cpus (group))); + +if (group->sgc->capacity !=SCHED_CAPACITY_SCALE ) { +seq_printf (m," (cpu_capacity = %d)" , + group->sgc->capacity); + } +seq_puts (m,"\n" ); + + group = group->next; + }while (group != sd->groups); + +if (!cpumask_equal (sched_domain_span (sd), &groupmask)) +seq_puts (m," ERROR: groups don't span domain->span\n" ); + +if (sd->parent && + !cpumask_subset (&groupmask,sched_domain_span (sd->parent))) { +seq_puts (m," ERROR: parent span is not a superset of domain->span\n" ); + } +} + +static int sched_cpu_show (struct seq_file *m,void *unused) +{ +struct sched_domain *sd; +int cpu = (int ) ((long ) m->private); + +if (cpu <0 || cpu >CONFIG_NR_CPUS ) { +seq_printf (m,"invalid CPU, %d\n" , cpu); +return 0 ; + } + +for_each_domain (cpu, sd) { +sched_cpu_domain_show (m, sd, cpu); +seq_puts (m,"\n" ); + } + +return 0 ; +} + +static int sched_cpu_open (struct inode *inode,struct file *filp) +{ +return single_open (filp, sched_cpu_show, inode->i_private); +} +static const struct file_operations sched_cpu_fops = { + .open = sched_cpu_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; +static struct dentry *d_sched_debug; +static struct dentry *d_sched_cpu[CONFIG_NR_CPUS ]; + +static int sched_debugfs_add_cpu (int cpu) +{ +char buf[32 ]; +long lcpu = cpu; + +snprintf (buf,sizeof (buf),"cpu%d" , cpu); + d_sched_cpu[cpu] =debugfs_create_file (buf,0444 , d_sched_debug, + (void *) lcpu, &sched_cpu_fops); + +if (d_sched_cpu[cpu] ==NULL ) +pr_warn ("Failed to create debugfs entry for cpu %d\n" , cpu); + +return 0 ; +} +static __init int sched_init_debug (void ) { -debugfs_create_file ("sched_features" ,0644 ,NULL ,NULL , +int cpu; +int rc =0 ; + + d_sched_debug =debugfs_create_dir ("sched" ,NULL ); +if (!d_sched_debug) { +pr_warn ("Could not create debugfs 'sched' entry\n" ); +return 0 ; + } + +debugfs_create_file ("features" ,0644 , d_sched_debug,NULL , &sched_feat_fops); -return 0 ; +for_each_online_cpu (cpu) { + rc =sched_debugfs_add_cpu (cpu); +if (rc) + goto out; + } + +out: +return rc; }late_initcall (sched_init_debug);#endif /* CONFIG_SCHED_DEBUG */
@@ -6689,7 +6850,7 @@ struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl,if (!cpumask_subset (sched_domain_span (child),sched_domain_span (sd))) { -pr_err ("BUG: arch topology borken\n" ); +pr_err ("BUG: arch topology broken\n" );#ifdef CONFIG_SCHED_DEBUG pr_err (" the %s domain not a subset of the %s domain\n" , child->name, sd->name);
Signature · 1 line
2.3.0
Quoted 16 lines · David Ahern
kernel/sched/core.c +8
-0@@ -268,12 +268,173 @@ static const struct file_operations sched_feat_fops = { .release = single_release, }; +static const char *const sd_flag_names[] = { +"load-balance" , +"new-idle" , +"exec" , +"fork" , +"wake" , +"affine" , +"" ,
kernel/sched/core.c static const struct file_operations sched_feat_fops = { · line 288
Quoted 10 lines
kernel/sched/core.c +10
-0+"cpu-capacity" , +"power-domain" , +"share-pkg-resources" , +"serialize" , +"asym-packing" , +"prefer-sibling" , +"overlap" , +"numa" , + "", +};
That's wrong with the names readers will want to grep for?
-Mike
Quoted 22 lines · Mike Galbraith
Quoted 20 lines · David Ahern
kernel/sched/core.c +12
-0@@ -268,12 +268,173 @@ static const struct file_operations sched_feat_fops = { .release = single_release, }; +static const char *const sd_flag_names[] = { +"load-balance" , +"new-idle" , +"exec" , +"fork" , +"wake" , +"affine" , +"" , +"cpu-capacity" , +"power-domain" , +"share-pkg-resources" , +"serialize" ,
Quoted 8 lines
kernel/sched/core.c static const struct file_operations sched_feat_fops = { · line 288
Quoted 6 lines
kernel/sched/core.c +6
-0+"asym-packing" , +"prefer-sibling" , +"overlap" , +"numa" , + "", +};That's wrong with the names readers will want to grep for?
I just found lower-case easier on the eyes. I could change this list to correspond to the SD_XXXXX defines. Certainly something that more programmatically correlates the macros (bit positions) and the names would be better.
David
Oh, you can add /proc/sys/kernel/sched_domain diddling to the hotplug todo... which will either create read /sys, poke modify buttons over yonder in /proc situation, or have to duplicate that.
-Mike
Quoted 4 lines · Mike Galbraith
Oh, you can add
/proc/sys/kernel/sched_domaindiddling to the hotplug todo... which will either create read /sys, poke modify buttons over yonder in /proc situation, or have to duplicate that.
-EPARSE.
you mean the /proc/sys/kernel/sched_domain/ files do not handle hotplug?
Can I get --verbose on what you mean above?
David
Quoted 11 lines · David Ahern
Quoted 4 lines · Mike Galbraith
Oh, you can add
/proc/sys/kernel/sched_domaindiddling to the hotplug todo... which will either create read /sys, poke modify buttons over yonder in /proc situation, or have to duplicate that.
-EPARSE.you mean the
/proc/sys/kernel/sched_domain/files do not handle hotplug?Can I get --verbose on what you mean above?
I meant hotplug isn't the only thing that alters what you display, and the display is disjointed from the knobs, user acquires pretty display in /sys, but the knobs to twiddle what he's looking at are elsewhere.
-Mike
Peter Zijlstra
maintainerQuoted 4 lines · David Ahern
Before spending too much time formalizing this I wanted to see if you guys would entertain the idea of making this info available via debugfs. It does move the existing
sched_featuresfile to sched/features -- not sure how acceptable it is to move files in debugfs.
There also already is a /proc/sys/kernel/sched_domain/ sysctl thing that has much of the stuff in.
I'm not opposed to a debugfs thing, but only if we can take the sysctl thing out.
Quoted 11 lines · Peter Zijlstra
Quoted 4 lines · David Ahern
Before spending too much time formalizing this I wanted to see if you guys would entertain the idea of making this info available via debugfs. It does move the existing
sched_featuresfile to sched/features -- not sure how acceptable it is to move files in debugfs.There also already is a
/proc/sys/kernel/sched_domain/sysctl thing that has much of the stuff in.I'm not opposed to a debugfs thing, but only if we can take the sysctl thing out.
Ew, I can see the regression reports now. That stuff has long roots.
-Mike
Peter Zijlstra
maintainerQuoted 7 lines · Mike Galbraith
Quoted 6 lines · Peter Zijlstra
Quoted 4 lines · David Ahern
Before spending too much time formalizing this I wanted to see if you guys would entertain the idea of making this info available via debugfs. It does move the existing
sched_featuresfile to sched/features -- not sure how acceptable it is to move files in debugfs.
Quoted 7 lines
Quoted 5 lines
There also already is a
/proc/sys/kernel/sched_domain/sysctl thing that has much of the stuff in.I'm not opposed to a debugfs thing, but only if we can take the sysctl thing out.
Ew, I can see the regression reports now. That stuff has long roots.
I didn't say we could take it out, I only said it would maybe be nice to be able to :-) But yah. It'll cause some pain if we do :/
Quoted 11 lines · Peter Zijlstra
Quoted 10 lines · Mike Galbraith
Quoted 9 lines · Peter Zijlstra
Quoted 4 lines · David Ahern
Before spending too much time formalizing this I wanted to see if you guys would entertain the idea of making this info available via debugfs. It does move the existing
sched_featuresfile to sched/features -- not sure how acceptable it is to move files in debugfs.There also already is a
/proc/sys/kernel/sched_domain/sysctl thing that has much of the stuff in.
Quoted 7 lines
Quoted 4 lines
Quoted 2 lines
I'm not opposed to a debugfs thing, but only if we can take the sysctl thing out.
Ew, I can see the regression reports now. That stuff has long roots.
I didn't say we could take it out, I only said it would maybe be nice to be able to :-) But yah. It'll cause some pain if we do :/
I think it would be a good thing if we can get away with it, but I also think you could safely bet your life that somebody will squeak.
-Mike
Peter Zijlstra
maintainerQuoted 2 lines · Mike Galbraith
I think it would be a good thing if we can get away with it, but I also think you could safely bet your life that somebody will squeak.
The thing I worry most about is that squeaking only happening 5 years later :/
Quoted 8 lines · Peter Zijlstra
Quoted 3 lines · Mike Galbraith
I think it would be a good thing if we can get away with it, but I also think you could safely bet your life that somebody will squeak.
The thing I worry most about is that squeaking only happening 5 years later :/
So lets start by keeping the sysctl thing with the very scheduler-internal names, but all zeroes and no effect of any change - i.e. a dead API in all but appearance. I don't think there's any legitimate use of those, beyond debugging, as we could change the internal implementation anymore and moot many of those flags.
So lets trigger the squeaking that way. If any complaint comes in beyond 1-2 kernel releases then I don't think it's a regression, it turns into a feature request ...
Thanks,
Ingo
Quoted 18 lines · Ingo Molnar
Quoted 8 lines · Peter Zijlstra
Quoted 3 lines · Mike Galbraith
I think it would be a good thing if we can get away with it, but I also think you could safely bet your life that somebody will squeak.
The thing I worry most about is that squeaking only happening 5 years later :/
So lets start by keeping the sysctl thing with the very scheduler-internal names, but all zeroes and no effect of any change - i.e. a dead API in all but appearance. I don't think there's any legitimate use of those, beyond debugging, as we could change the internal implementation anymore and moot many of those flags.
Quoted 3 lines
So lets trigger the squeaking that way. If any complaint comes in beyond 1-2 kernel releases then I don't think it's a regression, it turns into a feature request ...
Sorry to be so dense but I am not clear on what is acceptable and not. As mentioned in a previous response, these are the scheduler related files I am aware of:
- debufs file for sched_features (/sys/kernel/debug/sched_features)
- /proc/sys/kernel/sched_domain for tweaking scheduling parameters
- /proc/sched_debug - various internal stats
- /sys/devices/system/cpu entries. e.g., for cpu topology (physical package id, core id, sibling cores and threads)
None of them show the sched_domain information which is the subject of this patch.
Can someone clarify on the duplication concerns and such?
Thanks,
David
Quoted 2 lines · David Ahern
Can someone clarify on the duplication concerns and such?
My concern was eg you display flags in /sys, but to tweak flags etc, you must visit /proc. Seems to me the missing info should either be added to /proc (some under DEBUG?), or the lot should move to /sys.
I don't care all that much, making span etc visible anywhere is an win over boot flag + big box (esp. + serial console).
-Mike
Quoted 6 lines · Peter Zijlstra
Quoted 4 lines · David Ahern
Before spending too much time formalizing this I wanted to see if you guys would entertain the idea of making this info available via debugfs. It does move the existing
sched_featuresfile to sched/features -- not sure how acceptable it is to move files in debugfs.
Quoted 6 lines
There also already is a
/proc/sys/kernel/sched_domain/sysctl thing that has much of the stuff in.I'm not opposed to a debugfs thing, but only if we can take the sysctl thing out.
Related files I aware of; perhaps there are others:
- /sys/devices/system/cpu entries. e.g., for cpu topology (physical package id, core id, sibling cores and threads)
- debufs file for sched_features
- /proc/sys/kernel/sched_domain for tweaking scheduling parameters
I could not find anything that shows the sched_domain information. The proposal here was to create read-only debugfs files to dump the same info (but with better readability for large systems) that comes with the sched_debug parameter. I chose debugfs because a) sched_features is there and b) it is debug information about the scheduler.
I could just as easily out it under /proc/sys/kernel/sched_domain or create a sched subdir under /sys/devices/system/cpu/cpu$n if one of those is preferred.
David