cache pollution aware __copy_from_user_ll()
Subsystems (3)
A message that is not in the archive (sent off-list or not yet received): 20050818.201138.607962419.hyoshiok@miraclelinux.com.suse.lists.linux.kernel
A message that is not in the archive (sent off-list or not yet received): [email protected]
A message that is not in the archive (sent off-list or not yet received): 20050823.081246.846946371.hyoshiok@miraclelinux.com.suse.lists.linux.kernel
A message that is not in the archive (sent off-list or not yet received): 20050824.231156.278740508.hyoshiok@miraclelinux.com.suse.lists.linux.kernel
Quoted 6 lines · Hiro Yoshioka
Hi,
The following patch does not use MMX regsiters so that we don't have to worry about save/restore the FPU/MMX states.
What do you think?
Performance will probably be bad on K7 Athlons - those have a microcoded movnti which is quite slow.
Also BTW I don't see any code anywhere that tests the CPUID bits, so your code will fail spectacularly on a PII that didn't do SSE (intel user copy used to be enabled on those)
One way to solve this might be to use different code using alternative()
-Andi
From: Andi Kleen <[email protected]>
Quoted 10 lines
Quoted 6 lines
Hi,
The following patch does not use MMX regsiters so that we don't have to worry about save/restore the FPU/MMX states.
What do you think?
Performance will probably be bad on K7 Athlons - those have a microcoded movnti which is quite slow.
Quoted 8 lines
Also BTW I don't see any code anywhere that tests the CPUID bits, so your code will fail spectacularly on a PII that didn't do SSE (intel user copy used to be enabled on those)
One way to solve this might be to use different code using
alternative()-Andi
Thanks for your comments. I'll consider it.
Regards,
Hiro
Hi,
Quoted 19 lines
From: Andi Kleen <[email protected]>
Quoted 18 lines
Quoted 6 lines
Hi,
The following patch does not use MMX regsiters so that we don't have to worry about save/restore the FPU/MMX states.
What do you think?
Performance will probably be bad on K7 Athlons - those have a microcoded movnti which is quite slow.
Also BTW I don't see any code anywhere that tests the CPUID bits, so your code will fail spectacularly on a PII that didn't do SSE (intel user copy used to be enabled on those)
One way to solve this might be to use different code using
alternative()-Andi
The following is the almost final version of the cache pollution aware __copy_from_user_ll() patch.
1) use sfence instruction to perform a serializing on all store-to-memory instructions.
2) check if the cpu has the xmm2 extentions. (movnti)
I think it is a good enough to be considered into
the main line.
What do you think?
Some performance data are
Total of GLOBAL_POWER_EVENTS (CPU cycle samples)
2.6.12.4.orig 1921587
2.6.12.4.nt 1599424
1599424/1921587=83.23% (16.77% reduction)
BSQ_CACHE_REFERENCE (L3 cache miss)
2.6.12.4.orig 57427
2.6.12.4.nt 20858
20858/57427=36.32% (63.7% reduction)
L3 cache miss reduction of __copy_from_user_ll
samples %
37408 65.1412 vmlinux __copy_from_user_ll 23 0.1103 vmlinux __copy_user_zeroing_intel_nocache
23/37408=0.061% (99.94% reduction)
Top 5 of 2.6.12.4.nt
Counted GLOBAL_POWER_EVENTS events (time during which processor is not stopped) with a unit mask of 0x01 (mandatory) count 100000
samples % app name symbol name 128392 8.0274 vmlinux __copy_user_zeroing_intel_nocache 64206 4.0143 vmlinux journal_add_journal_head 59746 3.7355 vmlinux do_get_write_access 47674 2.9807 vmlinux journal_put_journal_head 46021 2.8774 vmlinux journal_dirty_metadata
pattern9-0-cpu4-0-09011728/summary.out
Counted BSQ_CACHE_REFERENCE events (cache references seen by the bus unit) with a unit mask of 0x3f (multiple flags) count 3000
samples % app name symbol name 69755 4.2861 vmlinux __copy_user_zeroing_intel_nocache 55685 3.4215 vmlinux journal_add_journal_head 52371 3.2179 vmlinux __find_get_block 45504 2.7960 vmlinux journal_put_journal_head 36005 2.2123 vmlinux journal_stop
pattern9-0-cpu4-0-09011744/summary.out
Counted BSQ_CACHE_REFERENCE events (cache references seen by the bus unit) with a unit mask of 0x200 (read 3rd level cache miss) count 3000
samples % app name symbol name 1147 5.4994 vmlinux journal_add_journal_head 881 4.2240 vmlinux journal_dirty_data 872 4.1809 vmlinux blk_rq_map_sg 734 3.5192 vmlinux journal_commit_transaction 617 2.9582 vmlinux radix_tree_delete
pattern9-0-cpu4-0-09011731/summary.out
Makefile +1 -1
@@ -1,7 +1,7 @@ VERSION = 2 PATCHLEVEL = 6 SUBLEVEL = 12 -EXTRAVERSION = .4.orig +EXTRAVERSION = .4.nt NAME=Woozy Numbat # *DOCUMENTATION*
arch/i386/lib/usercopy.c +118 -1
143 lines not on this page · raw message
include/asm-i386/uaccess.h +31 -0
@@ -413,6 +413,8 @@ const void *from, unsigned long n);unsigned long __must_check __copy_from_user_ll (void *to,const void __user *from,unsigned long n); +unsigned long __must_check __copy_from_user_ll_nocache (void *to, +const void __user *from,unsigned long n);/* * Here we special-case 1, 2 and 4-byte copy_*_user invocations. On a fault
@@ -502,11 +504,40 @@ }static inline unsigned long +__copy_from_user_inatomic_nocache (void *to,const void __user *from,unsigned long n) +{ +if (__builtin_constant_p (n)) { +unsigned long ret; + +switch (n) { +case 1 : +__get_user_size (*(u8 *)to, from,1 , ret,1 ); +return ret; +case 2 : +__get_user_size (*(u16 *)to, from,2 , ret,2 ); +return ret; +case 4 : +__get_user_size (*(u32 *)to, from,4 , ret,4 ); +return ret; + } + } +return __copy_from_user_ll_nocache (to, from, n); +} + +static inline unsigned long __copy_from_user (void *to,const void __user *from,unsigned long n) {might_sleep ();return __copy_from_user_inatomic (to, from, n); } + +static inline unsigned long +__copy_from_user_nocache (void *to,const void __user *from,unsigned long n) +{ +might_sleep (); +return __copy_from_user_inatomic_nocache (to, from, n); +} +unsigned long __must_check copy_to_user (void __user *to,const void *from,unsigned long n); unsigned long __must_check copy_from_user(void *to,
mm/filemap.c +3 -3
@@ -1727,13 +1727,13 @@int left; kaddr =kmap_atomic (page,KM_USER0 ); - left =__copy_from_user_inatomic (kaddr + offset, buf, bytes); + left =__copy_from_user_inatomic_nocache (kaddr + offset, buf, bytes);kunmap_atomic (kaddr,KM_USER0 );if (left !=0 ) {/* Do it the slow way */ kaddr =kmap (page); - left =__copy_from_user (kaddr + offset, buf, bytes); + left =__copy_from_user_nocache (kaddr + offset, buf, bytes);kunmap (page); }return bytes - left;
@@ -1750,7 +1750,7 @@int copy =min (bytes, iov->iov_len - base); base =0 ; - left =__copy_from_user_inatomic (vaddr, buf, copy); + left =__copy_from_user_inatomic_nocache (vaddr, buf, copy); copied += copy; bytes -= copy; vaddr += copy;
Regards,
Hiro
Quoted 2 lines · Hiro Yoshioka
The following is the almost final version of the cache pollution aware
__copy_from_user_ll()patch.
Looks good to me.
Once the filemap.c hunk is in I'll probably do something similar for x86-64.
-Andi
From: Andi Kleen <[email protected]>
Quoted 9 lines · Hiro Yoshioka
Quoted 3 lines
The following is the almost final version of the cache pollution aware
__copy_from_user_ll()patch.Looks good to me.
Once the
filemap.chunk is in I'll probably do something similar for x86-64.
Thank you very much. What else should I do? Shall I just be waiting to check in the patch?
Regards,
Hiro
Quoted 6 lines · Hiro Yoshioka
From: Andi Kleen <[email protected]>
Quoted 4 lines · Hiro Yoshioka
Quoted 2 lines
The following is the almost final version of the cache pollution aware
__copy_from_user_ll()patch.
Quoted 7 lines
Quoted 4 lines
Looks good to me.
Once the
filemap.chunk is in I'll probably do something similar for x86-64.Thank you very much. What else should I do? Shall I just be waiting to check in the patch?
I suppose Andrew will take care of it, unless someone else objects.
-Andi
Quoted 7 lines · Hiro Yoshioka
From: Andi Kleen <[email protected]>
Quoted 5 lines · Hiro Yoshioka
Quoted 3 lines
The following is the almost final version of the cache pollution aware
__copy_from_user_ll()patch.
Quoted 8 lines
Quoted 4 lines
Looks good to me.
Once the
filemap.chunk is in I'll probably do something similar for x86-64.Thank you very much. What else should I do? Shall I just be waiting to check in the patch?
I suppose I'll queue it up in -mm for a while, although I'm a bit dubious about the whole idea... We'll gain some and we'll lose some - how do we know it's a net gain?
Quoted 3 lines · Andrew Morton
I suppose I'll queue it up in -mm for a while, although I'm a bit dubious about the whole idea... We'll gain some and we'll lose some - how do we know it's a net gain?
I suspect it'll gain more than it loses. The only case where it might not gain is immediately someone reading the data from the page cache again after the write. But I suppose that's far less frequent than writing the data.
-Andi
Quoted 10 lines · Andi Kleen
Quoted 3 lines · Andrew Morton
I suppose I'll queue it up in -mm for a while, although I'm a bit dubious about the whole idea... We'll gain some and we'll lose some - how do we know it's a net gain?
I suspect it'll gain more than it loses. The only case where it might not gain is immediately someone reading the data from the page cache again after the write.
That's a pretty common case - temporary files.
Quoted 1 line
But I suppose that's far less frequent than writing the data.
yup.
Hiro, could you please send through a summary of the performance testing results sometime? Runtimes rather than oprofile output?
Andrew,
From: Andrew Morton <[email protected]>
Quoted 12 lines
Quoted 10 lines · Andi Kleen
Quoted 3 lines · Andrew Morton
I suppose I'll queue it up in -mm for a while, although I'm a bit dubious about the whole idea... We'll gain some and we'll lose some - how do we know it's a net gain?
I suspect it'll gain more than it loses. The only case where it might not gain is immediately someone reading the data from the page cache again after the write.
Quoted 8 lines
That's a pretty common case - temporary files.
Quoted 1 line
But I suppose that's far less frequent than writing the data.
yup.
Hiro, could you please send through a summary of the performance testing results sometime? Runtimes rather than oprofile output?
iozone results are
original 2.6.12.4 CPU time = 207.768 sec
cache aware CPU time = 184.783 sec
(three times run)
184.783/207.768=88.94% (11.06% reduction)
original:
pattern9-0-cpu4-0-08191720/iozone.out: CPU Utilization: Wall time 45.997 CPU time 64.527 CPU utilization 140.28 % pattern9-0-cpu4-0-08191741/iozone.out: CPU Utilization: Wall time 46.878 CPU time 71.933 CPU utilization 153.45 % pattern9-0-cpu4-0-08191743/iozone.out: CPU Utilization: Wall time 45.152 CPU time 71.308 CPU utilization 157.93 %
cache awre:
pattern9-0-cpu4-0-09011728/iozone.out: CPU Utilization: Wall time 44.842 CPU time 62.465 CPU utilization 139.30 % pattern9-0-cpu4-0-09011731/iozone.out: CPU Utilization: Wall time 44.718 CPU time 59.273 CPU utilization 132.55 % pattern9-0-cpu4-0-09011744/iozone.out: CPU Utilization: Wall time 44.367 CPU time 63.045 CPU utilization 142.10 %
Regards,
Hiro
Quoted 3 lines · Hiro Yoshioka
---
linux-2.6.12.4.orig/arch/i386/lib/usercopy.c2005-08-05 16:04:37.000000000 +0900
+++linux-2.6.12.4.nt/arch/i386/lib/usercopy.c2005-09-01 17:09:41.000000000 +0900
Really. Please redo and retest the patch against a current kernel.
From: Andrew Morton <[email protected]>
Quoted 6 lines
Quoted 3 lines · Hiro Yoshioka
---
linux-2.6.12.4.orig/arch/i386/lib/usercopy.c2005-08-05 16:04:37.000000000 +0900
+++linux-2.6.12.4.nt/arch/i386/lib/usercopy.c2005-09-01 17:09:41.000000000 +0900Really. Please redo and retest the patch against a current kernel.
Does it mean 2.6.13? I'll do it.
Regards,
Hiro
From: Hiro Yoshioka <[email protected]>
Subject: Re: [RFC] [PATCH] cache pollution aware __copy_from_user_ll()
Date: Fri, 02 Sep 2005 13:37:16 +0900 (JST)
Message-ID: <[email protected]>
Quoted 6 lines
From: Andrew Morton <[email protected]>
Quoted 5 lines
Quoted 3 lines · Hiro Yoshioka
---
linux-2.6.12.4.orig/arch/i386/lib/usercopy.c2005-08-05 16:04:37.000000000 +0900
+++linux-2.6.12.4.nt/arch/i386/lib/usercopy.c2005-09-01 17:09:41.000000000 +0900
Quoted 6 lines
Quoted 1 line
Really. Please redo and retest the patch against a current kernel.
Does it mean 2.6.13? I'll do it.
Regards,
Hiro
Hi,
The following is the patch against 2.6.13
Hiro
Makefile +1 -1
@@ -1,7 +1,7 @@ VERSION = 2 PATCHLEVEL = 6 SUBLEVEL = 13 -EXTRAVERSION = +EXTRAVERSION = .nt NAME=Woozy Numbat # *DOCUMENTATION*
arch/i386/lib/usercopy.c +118 -1
143 lines not on this page · raw message
include/asm-i386/uaccess.h +31 -0
@@ -413,6 +413,8 @@ const void *from, unsigned long n);unsigned long __must_check __copy_from_user_ll (void *to,const void __user *from,unsigned long n); +unsigned long __must_check __copy_from_user_ll_nocache (void *to, +const void __user *from,unsigned long n);/* * Here we special-case 1, 2 and 4-byte copy_*_user invocations. On a fault
@@ -502,11 +504,40 @@ }static inline unsigned long +__copy_from_user_inatomic_nocache (void *to,const void __user *from,unsigned long n) +{ +if (__builtin_constant_p (n)) { +unsigned long ret; + +switch (n) { +case 1 : +__get_user_size (*(u8 *)to, from,1 , ret,1 ); +return ret; +case 2 : +__get_user_size (*(u16 *)to, from,2 , ret,2 ); +return ret; +case 4 : +__get_user_size (*(u32 *)to, from,4 , ret,4 ); +return ret; + } + } +return __copy_from_user_ll_nocache (to, from, n); +} + +static inline unsigned long __copy_from_user (void *to,const void __user *from,unsigned long n) {might_sleep ();return __copy_from_user_inatomic (to, from, n); } + +static inline unsigned long +__copy_from_user_nocache (void *to,const void __user *from,unsigned long n) +{ +might_sleep (); +return __copy_from_user_inatomic_nocache (to, from, n); +} +unsigned long __must_check copy_to_user (void __user *to,const void *from,unsigned long n); unsigned long __must_check copy_from_user(void *to,
mm/filemap.c +1 -1
@@ -1726,7 +1726,7 @@int copy =min (bytes, iov->iov_len - base); base =0 ; - left =__copy_from_user_inatomic (vaddr, buf, copy); + left =__copy_from_user_inatomic_nocache (vaddr, buf, copy); copied += copy; bytes -= copy; vaddr += copy;
mm/filemap.h +2 -2
@@ -34,13 +34,13 @@int left; kaddr =kmap_atomic (page,KM_USER0 ); - left =__copy_from_user_inatomic (kaddr + offset, buf, bytes); + left =__copy_from_user_inatomic_nocache (kaddr + offset, buf, bytes);kunmap_atomic (kaddr,KM_USER0 );if (left !=0 ) {/* Do it the slow way */ kaddr =kmap (page); - left =__copy_from_user (kaddr + offset, buf, bytes); + left =__copy_from_user_nocache (kaddr + offset, buf, bytes);kunmap (page); }return bytes - left;