mm: vmstats: tlb flush counters
Subsystems (4)
PATCH RESEND 1/1 mm: vmstats: tlb flush counters
I was investigating some TLB flush scaling issues and realized that we do not have any good methods for figuring out how many TLB flushes we are doing.
It would be nice to be able to do these in generic code, but the arch-independent calls don't explicitly specify whether we actually need to do remote flushes or not. In the end, we really need to know if we actually _did_ global vs. local invalidations, so that leaves us with few options other than to muck with the counters from arch-specific code.
---
linux.git-davehans/arch/x86/mm/tlb.c | 18 ++++++++++++++---- linux.git-davehans/include/linux/vm_event_item.h | 5 +++++ linux.git-davehans/mm/vmstat.c | 5 +++++ 3 files changed, 24 insertions(+), 4 deletions(-)
arch/x86/mm/tlb.c renamed from arch/x86/mm/tlb.c~tlb-vmstats +14 -4
@@ -103,6 +103,7 @@ static void flush_tlb_func(void *info)if (f->flush_mm !=this_cpu_read (cpu_tlbstate.active_mm))return ; +count_vm_event (NR_TLB_REMOTE_FLUSH_RECEIVED );if (this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) {if (f->flush_end ==TLB_FLUSH_ALL )local_flush_tlb ();
@@ -130,6 +131,7 @@ void native_flush_tlb_others(const struc info.flush_start = start; info.flush_end = end; +count_vm_event (NR_TLB_REMOTE_FLUSH );if (is_uv_system ()) {unsigned int cpu;
@@ -149,6 +151,7 @@ void flush_tlb_current_task(void)preempt_disable (); +count_vm_event (NR_TLB_LOCAL_FLUSH_ALL );local_flush_tlb ();if (cpumask_any_but (mm_cpumask (mm),smp_processor_id ()) < nr_cpu_ids)flush_tlb_others (mm_cpumask (mm), mm,0UL ,TLB_FLUSH_ALL );
@@ -211,16 +214,19 @@ void flush_tlb_mm_range(struct mm_struct act_entries = mm->total_vm > tlb_entries ? tlb_entries : mm->total_vm;/* tlb_flushall_shift is on balance point, details in commit log */ -if ((end - start) >>PAGE_SHIFT > act_entries >> tlb_flushall_shift) +if ((end - start) >>PAGE_SHIFT > act_entries >> tlb_flushall_shift) { +count_vm_event (NR_TLB_LOCAL_FLUSH_ALL );local_flush_tlb (); -else { + }else {if (has_large_page (mm, start, end)) {local_flush_tlb (); goto flush_all; }/* flush range by one by one 'invlpg' */ -for (addr = start; addr < end; addr +=PAGE_SIZE ) +for (addr = start; addr < end; addr +=PAGE_SIZE ) { +count_vm_event (NR_TLB_LOCAL_FLUSH_ONE );__flush_tlb_single (addr); + }if (cpumask_any_but (mm_cpumask (mm),smp_processor_id ()) < nr_cpu_ids)
@@ -256,6 +262,7 @@ void flush_tlb_page(struct vm_area_strucstatic void do_flush_tlb_all (void *info) { +count_vm_event (NR_TLB_REMOTE_FLUSH_RECEIVED );__flush_tlb_all ();if (this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_LAZY )leave_mm (smp_processor_id ());
@@ -263,6 +270,7 @@ static void do_flush_tlb_all(void *info)void flush_tlb_all (void ) { +count_vm_event (NR_TLB_REMOTE_FLUSH );on_each_cpu (do_flush_tlb_all,NULL ,1 ); }
@@ -272,8 +280,10 @@ static void do_kernel_range_flush(void *unsigned long addr;/* flush range by one by one 'invlpg' */ -for (addr = f->flush_start; addr < f->flush_end; addr +=PAGE_SIZE ) +for (addr = f->flush_start; addr < f->flush_end; addr +=PAGE_SIZE ) { +count_vm_event (NR_TLB_LOCAL_FLUSH_ONE_KERNEL );__flush_tlb_single (addr); + } }void flush_tlb_kernel_range (unsigned long start,unsigned long end)
include/linux/vm_event_item.h renamed from include/linux/vm_event_item.h~tlb-vmstats +5 -0
@@ -70,6 +70,11 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSTHP_ZERO_PAGE_ALLOC ,THP_ZERO_PAGE_ALLOC_FAILED , #endif +NR_TLB_REMOTE_FLUSH ,/* cpu tried to flush others' tlbs */ +NR_TLB_REMOTE_FLUSH_RECEIVED ,/* cpu received ipi for flush */ +NR_TLB_LOCAL_FLUSH_ALL , +NR_TLB_LOCAL_FLUSH_ONE , +NR_TLB_LOCAL_FLUSH_ONE_KERNEL ,NR_VM_EVENT_ITEMS };
mm/vmstat.c renamed from mm/vmstat.c~tlb-vmstats +5 -0
@@ -817,6 +817,11 @@ const char * const vmstat_text[] = { "thp_zero_page_alloc", "thp_zero_page_alloc_failed", #endif + "nr_tlb_remote_flush", + "nr_tlb_remote_flush_received", + "nr_tlb_local_flush_all", + "nr_tlb_local_flush_one", + "nr_tlb_local_flush_one_kernel", #endif /* CONFIG_VM_EVENTS_COUNTERS */ };
_
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 101 lines · Dave Hansen
I was investigating some TLB flush scaling issues and realized that we do not have any good methods for figuring out how many TLB flushes we are doing.
It would be nice to be able to do these in generic code, but the arch-independent calls don't explicitly specify whether we actually need to do remote flushes or not. In the end, we really need to know if we actually _did_ global vs. local invalidations, so that leaves us with few options other than to muck with the counters from arch-specific code.
Signed-off-by Dave Hansen <[email protected]>---
linux.git-davehans/arch/x86/mm/tlb.c | 18 ++++++++++++++---- linux.git-davehans/include/linux/vm_event_item.h | 5 +++++ linux.git-davehans/mm/vmstat.c | 5 +++++ 3 files changed, 24 insertions(+), 4 deletions(-)diff -puN
arch/x86/mm/tlb.c~tlb-vmstatsarch/x86/mm/tlb.carch/x86/mm/tlb.c renamed from arch/x86/mm/tlb.c~tlb-vmstats +14
-4@@ -103,6 +103,7 @@ static void flush_tlb_func(void *info)if (f->flush_mm !=this_cpu_read (cpu_tlbstate.active_mm))return ; +count_vm_event (NR_TLB_REMOTE_FLUSH_RECEIVED );if (this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_OK ) {if (f->flush_end ==TLB_FLUSH_ALL )local_flush_tlb ();@@ -130,6 +131,7 @@ void native_flush_tlb_others(const struc info.flush_start = start; info.flush_end = end; +count_vm_event (NR_TLB_REMOTE_FLUSH );if (is_uv_system ()) {unsigned int cpu;@@ -149,6 +151,7 @@ void flush_tlb_current_task(void)preempt_disable (); +count_vm_event (NR_TLB_LOCAL_FLUSH_ALL );local_flush_tlb ();if (cpumask_any_but (mm_cpumask (mm),smp_processor_id ()) < nr_cpu_ids)flush_tlb_others (mm_cpumask (mm), mm,0UL ,TLB_FLUSH_ALL );@@ -211,16 +214,19 @@ void flush_tlb_mm_range(struct mm_struct act_entries = mm->total_vm > tlb_entries ? tlb_entries : mm->total_vm;/* tlb_flushall_shift is on balance point, details in commit log */ -if ((end - start) >>PAGE_SHIFT > act_entries >> tlb_flushall_shift) +if ((end - start) >>PAGE_SHIFT > act_entries >> tlb_flushall_shift) { +count_vm_event (NR_TLB_LOCAL_FLUSH_ALL );local_flush_tlb (); -else { + }else {if (has_large_page (mm, start, end)) {local_flush_tlb (); goto flush_all; }/* flush range by one by one 'invlpg' */ -for (addr = start; addr < end; addr +=PAGE_SIZE ) +for (addr = start; addr < end; addr +=PAGE_SIZE ) { +count_vm_event (NR_TLB_LOCAL_FLUSH_ONE );__flush_tlb_single (addr); + }if (cpumask_any_but (mm_cpumask (mm),smp_processor_id ()) < nr_cpu_ids)@@ -256,6 +262,7 @@ void flush_tlb_page(struct vm_area_strucstatic void do_flush_tlb_all (void *info) { +count_vm_event (NR_TLB_REMOTE_FLUSH_RECEIVED );__flush_tlb_all ();if (this_cpu_read (cpu_tlbstate.state) ==TLBSTATE_LAZY )leave_mm (smp_processor_id ());@@ -263,6 +270,7 @@ static void do_flush_tlb_all(void *info)void flush_tlb_all (void ) { +count_vm_event (NR_TLB_REMOTE_FLUSH );on_each_cpu (do_flush_tlb_all,NULL ,1 ); }@@ -272,8 +280,10 @@ static void do_kernel_range_flush(void *unsigned long addr;/* flush range by one by one 'invlpg' */ -for (addr = f->flush_start; addr < f->flush_end; addr +=PAGE_SIZE ) +for (addr = f->flush_start; addr < f->flush_end; addr +=PAGE_SIZE ) { +count_vm_event (NR_TLB_LOCAL_FLUSH_ONE_KERNEL );__flush_tlb_single (addr); + } }void flush_tlb_kernel_range (unsigned long start,unsigned long end)diff -puN
include/linux/vm_event_item.h~tlb-vmstatsinclude/linux/vm_event_item.h
include/linux/vm_event_item.h enum vm_event_item { PGPGIN, PGPGOUT, PS · line 78
Quoted 9 lines
include/linux/vm_event_item.h renamed from include/linux/vm_event_item.h~tlb-vmstats +5
-0@@ -70,6 +70,11 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSTHP_ZERO_PAGE_ALLOC ,THP_ZERO_PAGE_ALLOC_FAILED , #endif +NR_TLB_REMOTE_FLUSH ,/* cpu tried to flush others' tlbs */ +NR_TLB_REMOTE_FLUSH_RECEIVED ,/* cpu received ipi for flush */ +NR_TLB_LOCAL_FLUSH_ALL , +NR_TLB_LOCAL_FLUSH_ONE , + NR_TLB_LOCAL_FLUSH_ONE_KERNEL,
Please fix the vertical alignment of comments.
Quoted 7 lines
include/linux/vm_event_item.h renamed from include/linux/vm_event_item.h~tlb-vmstats +0
-0NR_VM_EVENT_ITEMS };diff -puN
mm/vmstat.c~tlb-vmstatsmm/vmstat.c
mm/vmstat.c const char * const vmstat_text[] = { · line 824
Quoted 8 lines
mm/vmstat.c renamed from mm/vmstat.c~tlb-vmstats +5
-0@@ -817,6 +817,11 @@ const char * const vmstat_text[] = {"thp_zero_page_alloc" ,"thp_zero_page_alloc_failed" , #endif +"nr_tlb_remote_flush" , +"nr_tlb_remote_flush_received" , +"nr_tlb_local_flush_all" , +"nr_tlb_local_flush_one" , + "nr_tlb_local_flush_one_kernel",
At first sight this seems pretty x86 specific. No range flush events, etc.
But no strong objections from me, if Andrew likes it.
Thanks,
Ingo
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Andrew Morton
maintainerQuoted 7 lines · On Wed, 17 Jul 2013 09:21:00 +0200 Ingo Molnar <[email protected]> wrote:
Quoted 4 lines · Dave Hansen
I was investigating some TLB flush scaling issues and realized that we do not have any good methods for figuring out how many TLB flushes we are doing.
Quoted 6 lines
Quoted 6 lines
It would be nice to be able to do these in generic code, but the arch-independent calls don't explicitly specify whether we actually need to do remote flushes or not. In the end, we really need to know if we actually _did_ global vs. local invalidations, so that leaves us with few options other than to muck with the counters from arch-specific code.
Spose so, if you really think it's worth it. It's all downside for uniprocessor machines. And for architectures which don't implement the counters, of course.
Quoted 7 lines
Quoted 7 lines
include/linux/vm_event_item.h renamed from include/linux/vm_event_item.h~tlb-vmstats +0
-0@@ -70,6 +70,11 @@ enum vm_event_item { PGPGIN, PGPGOUT, PS THP_ZERO_PAGE_ALLOC, THP_ZERO_PAGE_ALLOC_FAILED,#endif
Quoted 7 lines
include/linux/vm_event_item.h enum vm_event_item { PGPGIN, PGPGOUT, PS · line 77
Quoted 5 lines
include/linux/vm_event_item.h renamed from include/linux/vm_event_item.h~tlb-vmstats +5
-0+NR_TLB_REMOTE_FLUSH ,/* cpu tried to flush others' tlbs */ +NR_TLB_REMOTE_FLUSH_RECEIVED ,/* cpu received ipi for flush */ +NR_TLB_LOCAL_FLUSH_ALL , +NR_TLB_LOCAL_FLUSH_ONE , + NR_TLB_LOCAL_FLUSH_ONE_KERNEL,Please fix the vertical alignment of comments.
I looked - this isn't practical.
It would be nice to actually document these things though. We don't *have* to squeeze the comment into the RHS.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 16 lines · Andrew Morton
Quoted 7 lines · On Wed, 17 Jul 2013 09:21:00 +0200 Ingo Molnar <[email protected]> wrote:
Quoted 4 lines · Dave Hansen
I was investigating some TLB flush scaling issues and realized that we do not have any good methods for figuring out how many TLB flushes we are doing.
Quoted 6 lines
Quoted 6 lines
It would be nice to be able to do these in generic code, but the arch-independent calls don't explicitly specify whether we actually need to do remote flushes or not. In the end, we really need to know if we actually _did_ global vs. local invalidations, so that leaves us with few options other than to muck with the counters from arch-specific code.
Quoted 2 lines
Spose so, if you really think it's worth it. It's all downside for uniprocessor machines. [...]
UP is slowly going extinct, but in any case these counters ought to inform us about TLB flushes even on UP systems:
Quoted 3 lines
Quoted 3 lines
include/linux/vm_event_item.h enum vm_event_item { PGPGIN, PGPGOUT, PS · line 77
Quoted 3 lines
include/linux/vm_event_item.h renamed from include/linux/vm_event_item.h~tlb-vmstats +3
-0+NR_TLB_LOCAL_FLUSH_ALL , +NR_TLB_LOCAL_FLUSH_ONE , + NR_TLB_LOCAL_FLUSH_ONE_KERNEL,
While these ought to be compiled out on UP kernels:
Quoted 2 lines
Quoted 2 lines
include/linux/vm_event_item.h enum vm_event_item { PGPGIN, PGPGOUT, PS · line 74
Quoted 2 lines
include/linux/vm_event_item.h renamed from include/linux/vm_event_item.h~tlb-vmstats +2
-0+NR_TLB_REMOTE_FLUSH ,/* cpu tried to flush others' tlbs */ + NR_TLB_REMOTE_FLUSH_RECEIVED,/* cpu received ipi for flush */
Right?
Quoted 6 lines
Quoted 1 line
Please fix the vertical alignment of comments.
I looked - this isn't practical.
It would be nice to actually document these things though. We don't *have* to squeeze the comment into the RHS.
Agreed.
Thanks,
Ingo
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 11 lines · Ingo Molnar
UP is slowly going extinct, but in any case these counters ought to inform us about TLB flushes even on UP systems:
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
include/linux/vm_event_item.h enum vm_event_item { PGPGIN, PGPGOUT, PS · line 77
Quoted 3 lines
include/linux/vm_event_item.h renamed from include/linux/vm_event_item.h~tlb-vmstats +3
-0+NR_TLB_LOCAL_FLUSH_ALL , +NR_TLB_LOCAL_FLUSH_ONE , + NR_TLB_LOCAL_FLUSH_ONE_KERNEL,While these ought to be compiled out on UP kernels:
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
include/linux/vm_event_item.h enum vm_event_item { PGPGIN, PGPGOUT, PS · line 74
Quoted 2 lines
include/linux/vm_event_item.h renamed from include/linux/vm_event_item.h~tlb-vmstats +2
-0+NR_TLB_REMOTE_FLUSH ,/* cpu tried to flush others' tlbs */ + NR_TLB_REMOTE_FLUSH_RECEIVED,/* cpu received ipi for flush */Right?
Yeah, it's useful on UP too. But I realized that my changes were confined to the SMP code. The UP code is almost all in one of the headers, and I didn't touch it. So I've got some work there to fix it up.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 12 lines · Dave Hansen
I was investigating some TLB flush scaling issues and realized that we do not have any good methods for figuring out how many TLB flushes we are doing.
It would be nice to be able to do these in generic code, but the arch-independent calls don't explicitly specify whether we actually need to do remote flushes or not. In the end, we really need to know if we actually _did_ global vs. local invalidations, so that leaves us with few options other than to muck with the counters from arch-specific code.
Quoted 2 lines
Signed-off-by Dave Hansen <[email protected]>---
Hi Dave,
While measuring non - PLE performance, one of the bottleneck, I am seeing is flush tlbs.
perf had helped in alaysing a bit there, but this patch would help in precise calculation. It will aslo help in tuning the PLE window experiments (larger PLE window
would affect remote flush TLBs)
Thanks for this patch. Tested the patch on my sandybridge.box
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 6 lines · Raghavendra KT
While measuring non - PLE performance, one of the bottleneck, I am seeing is flush tlbs.
perf had helped in alaysing a bit there, but this patch would help in precise calculation. It will aslo help in tuning the PLE window experiments (larger PLE window
would affect remote flush TLBs)
Interesting. What workload is that? I've been having problems finding workloads that are too consumed with TLB flushes.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 11 lines · Dave Hansen
Quoted 6 lines · Raghavendra KT
While measuring non - PLE performance, one of the bottleneck, I am seeing is flush tlbs.
perf had helped in alaysing a bit there, but this patch would help in precise calculation. It will aslo help in tuning the PLE window experiments (larger PLE window
would affect remote flush TLBs)Interesting. What workload is that? I've been having problems finding workloads that are too consumed with TLB flushes.
Dave,
ebizzy is the one. and dbench to some small extent.
[root@codeblue ~]# cat /proc/vmstat |grep nr_tlb ; /root/data/script/do_ebizzy.sh; cat /proc/vmstat |grep nr_tlb nr_tlb_remote_flush 721nr_tlb_remote_flush_received 923nr_tlb_local_flush_all 13992nr_tlb_local_flush_one 0nr_tlb_local_flush_one_kernel 0
7482 records/s
real 120.00 s
user 86.69 s
sys 3746.57 snr_tlb_remote_flush 912896nr_tlb_remote_flush_received 28261974nr_tlb_local_flush_all 926272nr_tlb_local_flush_one 0nr_tlb_local_flush_one_kernel 0
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 10 lines · Dave Hansen
Quoted 6 lines · Raghavendra KT
While measuring non - PLE performance, one of the bottleneck, I am seeing is flush tlbs.
perf had helped in alaysing a bit there, but this patch would help in precise calculation. It will aslo help in tuning the PLE window experiments (larger PLE window
would affect remote flush TLBs)Interesting. What workload is that? I've been having problems finding workloads that are too consumed with TLB flushes.
Btw., would be nice to also integrate these VM counters into perf as well, as an instrumentation variant/option.
It could be done in an almost zero overhead fashion using jump-labels I think.
[ Just in case someone is bored to death and is looking for an interesting
side project ;-) ]
Thanks,
Ingo
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 8 lines · Ingo Molnar
Btw., would be nice to also integrate these VM counters into perf as well, as an instrumentation variant/option.
It could be done in an almost zero overhead fashion using jump-labels I think.
[ Just in case someone is bored to death and is looking for an interesting
side project ;-) ]
I'd actually been thinking about making them in to tracepoints, but the tracepoint macros seem to create #include messes if you try to use them in very common headers.
Agree it would be an interesting side project, though. :)
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 7 lines · Dave Hansen
Quoted 6 lines · Ingo Molnar
Btw., would be nice to also integrate these VM counters into perf as well, as an instrumentation variant/option.
It could be done in an almost zero overhead fashion using jump-labels I think.
Quoted 8 lines
Quoted 2 lines
[ Just in case someone is bored to death and is looking for an interesting
side project ;-) ]I'd actually been thinking about making them in to tracepoints, but the tracepoint macros seem to create #include messes if you try to use them in very common headers.
Agree it would be an interesting side project, though. :)
Yes, tracepoints was what I was thinking about, it would allow easy integration into perf [and it's useful even without any userspace side] - as long as:
- the tracepoints trace the counts/sums, not just the events themselves
- when the tracepoints are not active the VM counts are still maintained separately
I.e. the existing VM counts and its extraction facilities are not impacted in any way, just a new channel of instrumentation is provided - jump-label/static-key optimized by virtue of being tracepoints.
Thanks,
Ingo
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>