Skip to contentSkip to conversation

cache pollution aware __copy_from_user_ll()

RFC PATCH Andi Kleen 3 participants 13 messages Aug 24, 2005 – Sep 3, 2005 4 messages not in the archive

Subsystems (3)
lore
Share & tools
lore

Andi Kleen

author
#4 lore raw
In reply to a message not in the archive
Quoted 6 lines · Hiro Yoshioka

Hi,

The following patch does not use MMX regsiters so that we don't have to worry about save/restore the FPU/MMX states.

What do you think?

Performance will probably be bad on K7 Athlons - those have a microcoded movnti which is quite slow.

Also BTW I don't see any code anywhere that tests the CPUID bits, so your code will fail spectacularly on a PII that didn't do SSE (intel user copy used to be enabled on those)

One way to solve this might be to use different code using alternative()

-Andi

#5 lore raw

From: Andi Kleen <[email protected]>

Quoted 10 lines
Quoted 6 lines

Hi,

The following patch does not use MMX regsiters so that we don't have to worry about save/restore the FPU/MMX states.

What do you think?

Performance will probably be bad on K7 Athlons - those have a microcoded movnti which is quite slow.

Quoted 8 lines

Also BTW I don't see any code anywhere that tests the CPUID bits, so your code will fail spectacularly on a PII that didn't do SSE (intel user copy used to be enabled on those)

One way to solve this might be to use different code using alternative()

-Andi

Thanks for your comments. I'll consider it.

Regards,
Hiro

#6 lore raw

Hi,

Quoted 19 lines

From: Andi Kleen <[email protected]>

Quoted 18 lines
Quoted 6 lines

Hi,

The following patch does not use MMX regsiters so that we don't have to worry about save/restore the FPU/MMX states.

What do you think?

Performance will probably be bad on K7 Athlons - those have a microcoded movnti which is quite slow.

Also BTW I don't see any code anywhere that tests the CPUID bits, so your code will fail spectacularly on a PII that didn't do SSE (intel user copy used to be enabled on those)

One way to solve this might be to use different code using alternative()

-Andi

The following is the almost final version of the cache pollution aware __copy_from_user_ll() patch.

1) use sfence instruction to perform a serializing on all store-to-memory instructions.
2) check if the cpu has the xmm2 extentions. (movnti)

I think it is a good enough to be considered into
the main line.

What do you think?

Some performance data are

Total of GLOBAL_POWER_EVENTS (CPU cycle samples)

2.6.12.4.orig 1921587
2.6.12.4.nt 1599424
1599424/1921587=83.23% (16.77% reduction)

BSQ_CACHE_REFERENCE (L3 cache miss)
2.6.12.4.orig 57427
2.6.12.4.nt 20858
20858/57427=36.32% (63.7% reduction)

L3 cache miss reduction of __copy_from_user_ll
samples %

37408    65.1412  vmlinux                  __copy_from_user_ll
23        0.1103  vmlinux                  __copy_user_zeroing_intel_nocache

23/37408=0.061% (99.94% reduction)

Top 5 of 2.6.12.4.nt
Counted GLOBAL_POWER_EVENTS events (time during which processor is not stopped) with a unit mask of 0x01 (mandatory) count 100000

samples  %        app name                 symbol name
128392    8.0274  vmlinux                  __copy_user_zeroing_intel_nocache
64206     4.0143  vmlinux                  journal_add_journal_head
59746     3.7355  vmlinux                  do_get_write_access
47674     2.9807  vmlinux                  journal_put_journal_head
46021     2.8774  vmlinux                  journal_dirty_metadata

pattern9-0-cpu4-0-09011728/summary.out

Counted BSQ_CACHE_REFERENCE events (cache references seen by the bus unit) with a unit mask of 0x3f (multiple flags) count 3000

samples  %        app name                 symbol name
69755     4.2861  vmlinux                  __copy_user_zeroing_intel_nocache
55685     3.4215  vmlinux                  journal_add_journal_head
52371     3.2179  vmlinux                  __find_get_block
45504     2.7960  vmlinux                  journal_put_journal_head
36005     2.2123  vmlinux                  journal_stop

pattern9-0-cpu4-0-09011744/summary.out

Counted BSQ_CACHE_REFERENCE events (cache references seen by the bus unit) with a unit mask of 0x200 (read 3rd level cache miss) count 3000

samples  %        app name                 symbol name
1147      5.4994  vmlinux                  journal_add_journal_head
881       4.2240  vmlinux                  journal_dirty_data
872       4.1809  vmlinux                  blk_rq_map_sg
734       3.5192  vmlinux                  journal_commit_transaction
617       2.9582  vmlinux                  radix_tree_delete

pattern9-0-cpu4-0-09011731/summary.out

Makefile +1 -1
diff -ur linux-2.6.12.4.orig/Makefile linux-2.6.12.4.nt/Makefile
@@ -1,7 +1,7 @@
 VERSION = 2
 PATCHLEVEL = 6
 SUBLEVEL = 12
-EXTRAVERSION = .4.orig
+EXTRAVERSION = .4.nt
 NAME=Woozy Numbat
 
 # *DOCUMENTATION*
arch/i386/lib/usercopy.c +118 -1

143 lines not on this page · raw message

include/asm-i386/uaccess.h +31 -0
diff -ur linux-2.6.12.4.orig/include/asm-i386/uaccess.h linux-2.6.12.4.nt/include/asm-i386/uaccess.h
@@ -413,6 +413,8 @@
 				const void *from, unsigned long n);
 unsigned long __must_check __copy_from_user_ll(void *to,
 				const void __user *from, unsigned long n);
+unsigned long __must_check __copy_from_user_ll_nocache(void *to,
+				const void __user *from, unsigned long n);
 
 /*
  * Here we special-case 1, 2 and 4-byte copy_*_user invocations.  On a fault
@@ -502,11 +504,40 @@
 }
 
 static inline unsigned long
+__copy_from_user_inatomic_nocache(void *to, const void __user *from, unsigned long n)
+{
+	if (__builtin_constant_p(n)) {
+		unsigned long ret;
+
+		switch (n) {
+		case 1:
+			__get_user_size(*(u8 *)to, from, 1, ret, 1);
+			return ret;
+		case 2:
+			__get_user_size(*(u16 *)to, from, 2, ret, 2);
+			return ret;
+		case 4:
+			__get_user_size(*(u32 *)to, from, 4, ret, 4);
+			return ret;
+		}
+	}
+	return __copy_from_user_ll_nocache(to, from, n);
+}
+
+static inline unsigned long
 __copy_from_user(void *to, const void __user *from, unsigned long n)
 {
        might_sleep();
        return __copy_from_user_inatomic(to, from, n);
 }
+
+static inline unsigned long
+__copy_from_user_nocache(void *to, const void __user *from, unsigned long n)
+{
+       might_sleep();
+       return __copy_from_user_inatomic_nocache(to, from, n);
+}
+
 unsigned long __must_check copy_to_user(void __user *to,
 				const void *from, unsigned long n);
 unsigned long __must_check copy_from_user(void *to,
mm/filemap.c +3 -3
diff -ur linux-2.6.12.4.orig/mm/filemap.c linux-2.6.12.4.nt/mm/filemap.c
@@ -1727,13 +1727,13 @@
 	int left;
 
 	kaddr = kmap_atomic(page, KM_USER0);
-	left = __copy_from_user_inatomic(kaddr + offset, buf, bytes);
+	left = __copy_from_user_inatomic_nocache(kaddr + offset, buf, bytes);
 	kunmap_atomic(kaddr, KM_USER0);
 
 	if (left != 0) {
 		/* Do it the slow way */
 		kaddr = kmap(page);
-		left = __copy_from_user(kaddr + offset, buf, bytes);
+		left = __copy_from_user_nocache(kaddr + offset, buf, bytes);
 		kunmap(page);
 	}
 	return bytes - left;
@@ -1750,7 +1750,7 @@
 		int copy = min(bytes, iov->iov_len - base);
 
 		base = 0;
-		left = __copy_from_user_inatomic(vaddr, buf, copy);
+		left = __copy_from_user_inatomic_nocache(vaddr, buf, copy);
 		copied += copy;
 		bytes -= copy;
 		vaddr += copy;

Regards,
Hiro

author
#7 lore raw
Quoted 2 lines · Hiro Yoshioka

The following is the almost final version of the cache pollution aware __copy_from_user_ll() patch.

Looks good to me.

Once the filemap.c hunk is in I'll probably do something similar for x86-64.

-Andi

#8 lore raw

From: Andi Kleen <[email protected]>

Quoted 9 lines · Hiro Yoshioka
Quoted 3 lines

The following is the almost final version of the cache pollution aware __copy_from_user_ll() patch.

Looks good to me.

Once the filemap.c hunk is in I'll probably do something similar for x86-64.

Thank you very much. What else should I do? Shall I just be waiting to check in the patch?

Regards,
Hiro

author
#9 lore raw
Quoted 6 lines · Hiro Yoshioka

From: Andi Kleen <[email protected]>

Quoted 4 lines · Hiro Yoshioka
Quoted 2 lines

The following is the almost final version of the cache pollution aware __copy_from_user_ll() patch.

Quoted 7 lines
Quoted 4 lines

Looks good to me.

Once the filemap.c hunk is in I'll probably do something similar for x86-64.

Thank you very much. What else should I do? Shall I just be waiting to check in the patch?

I suppose Andrew will take care of it, unless someone else objects.

-Andi

#10 lore raw
In reply to Hiro Yoshioka
Quoted 7 lines · Hiro Yoshioka

From: Andi Kleen <[email protected]>

Quoted 5 lines · Hiro Yoshioka
Quoted 3 lines

The following is the almost final version of the cache pollution aware __copy_from_user_ll() patch.

Quoted 8 lines
Quoted 4 lines

Looks good to me.

Once the filemap.c hunk is in I'll probably do something similar for x86-64.

Thank you very much. What else should I do? Shall I just be waiting to check in the patch?

I suppose I'll queue it up in -mm for a while, although I'm a bit dubious about the whole idea... We'll gain some and we'll lose some - how do we know it's a net gain?

author
#11 lore raw
Quoted 3 lines · Andrew Morton

I suppose I'll queue it up in -mm for a while, although I'm a bit dubious about the whole idea... We'll gain some and we'll lose some - how do we know it's a net gain?

I suspect it'll gain more than it loses. The only case where it might not gain is immediately someone reading the data from the page cache again after the write. But I suppose that's far less frequent than writing the data.

-Andi

#12 lore raw
Quoted 10 lines · Andi Kleen
Quoted 3 lines · Andrew Morton

I suppose I'll queue it up in -mm for a while, although I'm a bit dubious about the whole idea... We'll gain some and we'll lose some - how do we know it's a net gain?

I suspect it'll gain more than it loses. The only case where it might not gain is immediately someone reading the data from the page cache again after the write.

That's a pretty common case - temporary files.

Quoted 1 line

But I suppose that's far less frequent than writing the data.

yup.

Hiro, could you please send through a summary of the performance testing results sometime? Runtimes rather than oprofile output?

#13 lore raw

Andrew,

From: Andrew Morton <[email protected]>

Quoted 12 lines
Quoted 10 lines · Andi Kleen
Quoted 3 lines · Andrew Morton

I suppose I'll queue it up in -mm for a while, although I'm a bit dubious about the whole idea... We'll gain some and we'll lose some - how do we know it's a net gain?

I suspect it'll gain more than it loses. The only case where it might not gain is immediately someone reading the data from the page cache again after the write.

Quoted 8 lines

That's a pretty common case - temporary files.

Quoted 1 line

But I suppose that's far less frequent than writing the data.

yup.

Hiro, could you please send through a summary of the performance testing results sometime? Runtimes rather than oprofile output?

iozone results are

original 2.6.12.4 CPU time = 207.768 sec
cache aware CPU time = 184.783 sec
(three times run)
184.783/207.768=88.94% (11.06% reduction)

original:

pattern9-0-cpu4-0-08191720/iozone.out:  CPU Utilization: Wall time   45.997    CPU time   64.527    CPU utilization 140.28 %
pattern9-0-cpu4-0-08191741/iozone.out:  CPU Utilization: Wall time   46.878    CPU time   71.933    CPU utilization 153.45 %
pattern9-0-cpu4-0-08191743/iozone.out:  CPU Utilization: Wall time   45.152    CPU time   71.308    CPU utilization 157.93 %

cache awre:

pattern9-0-cpu4-0-09011728/iozone.out:  CPU Utilization: Wall time   44.842    CPU time   62.465    CPU utilization 139.30 %
pattern9-0-cpu4-0-09011731/iozone.out:  CPU Utilization: Wall time   44.718    CPU time   59.273    CPU utilization 132.55 %
pattern9-0-cpu4-0-09011744/iozone.out:  CPU Utilization: Wall time   44.367    CPU time   63.045    CPU utilization 142.10 %

Regards,
Hiro

#14 lore raw
In reply to Hiro Yoshioka
Quoted 3 lines · Hiro Yoshioka

--- linux-2.6.12.4.orig/arch/i386/lib/usercopy.c 2005-08-05 16:04:37.000000000 +0900
+++ linux-2.6.12.4.nt/arch/i386/lib/usercopy.c 2005-09-01 17:09:41.000000000 +0900

Really. Please redo and retest the patch against a current kernel.

#15 lore raw

From: Andrew Morton <[email protected]>

Quoted 6 lines
Quoted 3 lines · Hiro Yoshioka

--- linux-2.6.12.4.orig/arch/i386/lib/usercopy.c 2005-08-05 16:04:37.000000000 +0900
+++ linux-2.6.12.4.nt/arch/i386/lib/usercopy.c 2005-09-01 17:09:41.000000000 +0900

Really. Please redo and retest the patch against a current kernel.

Does it mean 2.6.13? I'll do it.

Regards,
Hiro

#16 lore raw

From: Hiro Yoshioka <[email protected]>
Subject: Re: [RFC] [PATCH] cache pollution aware __copy_from_user_ll()
Date: Fri, 02 Sep 2005 13:37:16 +0900 (JST)
Message-ID: <[email protected]>

Quoted 6 lines

From: Andrew Morton <[email protected]>

Quoted 5 lines
Quoted 3 lines · Hiro Yoshioka

--- linux-2.6.12.4.orig/arch/i386/lib/usercopy.c 2005-08-05 16:04:37.000000000 +0900
+++ linux-2.6.12.4.nt/arch/i386/lib/usercopy.c 2005-09-01 17:09:41.000000000 +0900

Quoted 6 lines
Quoted 1 line

Really. Please redo and retest the patch against a current kernel.

Does it mean 2.6.13? I'll do it.

Regards,
Hiro

Hi,

The following is the patch against 2.6.13

Hiro

Makefile +1 -1
diff -ur linux-2.6.13/Makefile linux-2.6.13.nt/Makefile
@@ -1,7 +1,7 @@
 VERSION = 2
 PATCHLEVEL = 6
 SUBLEVEL = 13
-EXTRAVERSION =
+EXTRAVERSION = .nt
 NAME=Woozy Numbat
 
 # *DOCUMENTATION*
arch/i386/lib/usercopy.c +118 -1

143 lines not on this page · raw message

include/asm-i386/uaccess.h +31 -0
diff -ur linux-2.6.13/include/asm-i386/uaccess.h linux-2.6.13.nt/include/asm-i386/uaccess.h
@@ -413,6 +413,8 @@
 				const void *from, unsigned long n);
 unsigned long __must_check __copy_from_user_ll(void *to,
 				const void __user *from, unsigned long n);
+unsigned long __must_check __copy_from_user_ll_nocache(void *to,
+				const void __user *from, unsigned long n);
 
 /*
  * Here we special-case 1, 2 and 4-byte copy_*_user invocations.  On a fault
@@ -502,11 +504,40 @@
 }
 
 static inline unsigned long
+__copy_from_user_inatomic_nocache(void *to, const void __user *from, unsigned long n)
+{
+	if (__builtin_constant_p(n)) {
+		unsigned long ret;
+
+		switch (n) {
+		case 1:
+			__get_user_size(*(u8 *)to, from, 1, ret, 1);
+			return ret;
+		case 2:
+			__get_user_size(*(u16 *)to, from, 2, ret, 2);
+			return ret;
+		case 4:
+			__get_user_size(*(u32 *)to, from, 4, ret, 4);
+			return ret;
+		}
+	}
+	return __copy_from_user_ll_nocache(to, from, n);
+}
+
+static inline unsigned long
 __copy_from_user(void *to, const void __user *from, unsigned long n)
 {
        might_sleep();
        return __copy_from_user_inatomic(to, from, n);
 }
+
+static inline unsigned long
+__copy_from_user_nocache(void *to, const void __user *from, unsigned long n)
+{
+       might_sleep();
+       return __copy_from_user_inatomic_nocache(to, from, n);
+}
+
 unsigned long __must_check copy_to_user(void __user *to,
 				const void *from, unsigned long n);
 unsigned long __must_check copy_from_user(void *to,
mm/filemap.c +1 -1
diff -ur linux-2.6.13/mm/filemap.c linux-2.6.13.nt/mm/filemap.c
@@ -1726,7 +1726,7 @@
 		int copy = min(bytes, iov->iov_len - base);
 
 		base = 0;
-		left = __copy_from_user_inatomic(vaddr, buf, copy);
+		left = __copy_from_user_inatomic_nocache(vaddr, buf, copy);
 		copied += copy;
 		bytes -= copy;
 		vaddr += copy;
mm/filemap.h +2 -2
diff -ur linux-2.6.13/mm/filemap.h linux-2.6.13.nt/mm/filemap.h
@@ -34,13 +34,13 @@
 	int left;
 
 	kaddr = kmap_atomic(page, KM_USER0);
-	left = __copy_from_user_inatomic(kaddr + offset, buf, bytes);
+	left = __copy_from_user_inatomic_nocache(kaddr + offset, buf, bytes);
 	kunmap_atomic(kaddr, KM_USER0);
 
 	if (left != 0) {
 		/* Do it the slow way */
 		kaddr = kmap(page);
-		left = __copy_from_user(kaddr + offset, buf, bytes);
+		left = __copy_from_user_nocache(kaddr + offset, buf, bytes);
 		kunmap(page);
 	}
 	return bytes - left;