Skip to contentSkip to conversation

powerpc: optimise smp_mb

PATCH In review Nick Piggin 2 participants 6 messages Feb 19, 2009 – Mar 4, 2009

Subsystems (4)
lore
Share & tools
lore
#0 lore raw

PATCH 1/2 powerpc: optimise smp_mb

Using lwsync, isync sequence in a microbenchmark is 5 times faster on my G5 than using sync for smp_mb. Although it takes more instructions.

Running tbench with 4 clients on my 4 core G5 (20 times) gives the following:

unpatched AVG=920.33 STD=2.36
patched AVG=921.27 STD=2.77

So not a big improvement here, actually it could even be in the noise. But other workloads or systems might see a bigger win, and the patch maybe is interesting or could be improved, so I'll ask for comments.

---

arch/powerpc/include/asm/system.h +9 -0
Index: linux-2.6/arch/powerpc/include/asm/system.h
===================================================================
@@ -52,7 +52,16 @@
 #    define SMPWMB      eieio
 #endif
 
+#ifdef __powerpc64__
+#define smp_mb()	__asm__ __volatile__ (				    \
+					"1:	lwsync			\n" \
+					"	cmpw	0,%%r0,%%r0	\n" \
+					"	bne-	1b		\n" \
+					"	isync			\n" \
+					: : : "memory")
+#else
 #define smp_mb()	mb()
+#endif
 #define smp_rmb()	__asm__ __volatile__ (stringify_in_c(LWSYNC) : : :"memory")
 #define smp_wmb()	__asm__ __volatile__ (stringify_in_c(SMPWMB) : : :"memory")
 #define smp_read_barrier_depends()	read_barrier_depends()
#1 lore raw

PATCH 2/2 powerpc: replace isync with lwsync

OK, here is this patch again. You didn't think I'd let a 2% performance improvement be forgotten? :)

Anyway, patch won't work well on architecture without lwsync, but I won't bother fixing that kind of thing and making it merge worthy until you guys say something positive about it.

20 runs of tbench on the G5

unpatched AVG=920.37 STD=2.36
patched AVG=938.89 STD=3.33

(throughput in MB/s) This is a 1.9% throughput increase.

---

arch/powerpc/include/asm/atomic.h +12 -12
Index: linux-2.6/arch/powerpc/include/asm/atomic.h
===================================================================
@@ -55,7 +55,7 @@
 	PPC405_ERR77(0,%2)
 "	stwcx.	%0,0,%2 \n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (t)
 	: "r" (a), "r" (&v->counter)
 	: "cc", "memory");
@@ -91,7 +91,7 @@
 	PPC405_ERR77(0,%2)
 "	stwcx.	%0,0,%2 \n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (t)
 	: "r" (a), "r" (&v->counter)
 	: "cc", "memory");
@@ -125,7 +125,7 @@
 	PPC405_ERR77(0,%1)
 "	stwcx.	%0,0,%1 \n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (t)
 	: "r" (&v->counter)
 	: "cc", "xer", "memory");
@@ -169,7 +169,7 @@
 	PPC405_ERR77(0,%1)
 "	stwcx.	%0,0,%1\n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (t)
 	: "r" (&v->counter)
 	: "cc", "xer", "memory");
@@ -202,7 +202,7 @@
 	PPC405_ERR77(0,%2)
 "	stwcx.	%0,0,%1 \n\
 	bne-	1b \n"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 "	subf	%0,%2,%0 \n\
 2:"
 	: "=&r" (t)
@@ -235,7 +235,7 @@
 	PPC405_ERR77(0,%1)
 "	stwcx.	%0,0,%1\n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	"\n\
 2:"	: "=&b" (t)
 	: "r" (&v->counter)
@@ -291,7 +291,7 @@
 	add	%0,%1,%0\n\
 	stdcx.	%0,0,%2 \n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (t)
 	: "r" (a), "r" (&v->counter)
 	: "cc", "memory");
@@ -325,7 +325,7 @@
 	subf	%0,%1,%0\n\
 	stdcx.	%0,0,%2 \n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (t)
 	: "r" (a), "r" (&v->counter)
 	: "cc", "memory");
@@ -357,7 +357,7 @@
 	addic	%0,%0,1\n\
 	stdcx.	%0,0,%1 \n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (t)
 	: "r" (&v->counter)
 	: "cc", "xer", "memory");
@@ -399,7 +399,7 @@
 	addic	%0,%0,-1\n\
 	stdcx.	%0,0,%1\n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (t)
 	: "r" (&v->counter)
 	: "cc", "xer", "memory");
@@ -425,7 +425,7 @@
 	blt-	2f\n\
 	stdcx.	%0,0,%1\n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	"\n\
 2:"	: "=&r" (t)
 	: "r" (&v->counter)
@@ -458,7 +458,7 @@
 	add	%0,%2,%0 \n"
 "	stdcx.	%0,0,%1 \n\
 	bne-	1b \n"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 "	subf	%0,%2,%0 \n\
 2:"
 	: "=&r" (t)
arch/powerpc/include/asm/bitops.h +4 -4
Index: linux-2.6/arch/powerpc/include/asm/bitops.h
===================================================================
@@ -139,7 +139,7 @@
 	PPC405_ERR77(0,%3)
 	PPC_STLCX "%1,0,%3 \n"
 	"bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (old), "=&r" (t)
 	: "r" (mask), "r" (p)
 	: "cc", "memory");
@@ -160,7 +160,7 @@
 	PPC405_ERR77(0,%3)
 	PPC_STLCX "%1,0,%3 \n"
 	"bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (old), "=&r" (t)
 	: "r" (mask), "r" (p)
 	: "cc", "memory");
@@ -182,7 +182,7 @@
 	PPC405_ERR77(0,%3)
 	PPC_STLCX "%1,0,%3 \n"
 	"bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (old), "=&r" (t)
 	: "r" (mask), "r" (p)
 	: "cc", "memory");
@@ -204,7 +204,7 @@
 	PPC405_ERR77(0,%3)
 	PPC_STLCX "%1,0,%3 \n"
 	"bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (old), "=&r" (t)
 	: "r" (mask), "r" (p)
 	: "cc", "memory");
arch/powerpc/include/asm/futex.h +1 -1
Index: linux-2.6/arch/powerpc/include/asm/futex.h
===================================================================
@@ -97,7 +97,7 @@
         PPC405_ERR77(0,%2)
 "2:     stwcx.  %4,0,%2\n\
         bne-    1b\n"
-        ISYNC_ON_SMP
+        LWSYNC_ON_SMP
 "3:	.section .fixup,\"ax\"\n\
 4:	li	%0,%5\n\
 	b	3b\n\
arch/powerpc/include/asm/spinlock.h +3 -3
Index: linux-2.6/arch/powerpc/include/asm/spinlock.h
===================================================================
@@ -65,7 +65,7 @@
 	bne-		2f\n\
 	stwcx.		%1,0,%2\n\
 	bne-		1b\n\
-	isync\n\
+	lwsync\n\
 2:"	: "=&r" (tmp)
 	: "r" (token), "r" (&lock->slock)
 	: "cr0", "memory");
@@ -193,7 +193,7 @@
 	PPC405_ERR77(0,%1)
 "	stwcx.		%0,0,%1\n\
 	bne-		1b\n\
-	isync\n\
+	lwsync\n\
 2:"	: "=&r" (tmp)
 	: "r" (&rw->lock)
 	: "cr0", "xer", "memory");
@@ -217,7 +217,7 @@
 	PPC405_ERR77(0,%1)
 "	stwcx.		%1,0,%2\n\
 	bne-		1b\n\
-	isync\n\
+	lwsync\n\
 2:"	: "=&r" (tmp)
 	: "r" (token), "r" (&rw->lock)
 	: "cr0", "memory");
arch/powerpc/include/asm/system.h +4 -4
Index: linux-2.6/arch/powerpc/include/asm/system.h
===================================================================
@@ -246,7 +246,7 @@
 	PPC405_ERR77(0,%2)
 "	stwcx.	%3,0,%2 \n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (prev), "+m" (*(volatile unsigned int *)p)
 	: "r" (p), "r" (val)
 	: "cc", "memory");
@@ -289,7 +289,7 @@
 	PPC405_ERR77(0,%2)
 "	stdcx.	%3,0,%2 \n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (prev), "+m" (*(volatile unsigned long *)p)
 	: "r" (p), "r" (val)
 	: "cc", "memory");
@@ -382,7 +382,7 @@
 	PPC405_ERR77(0,%2)
 "	stwcx.	%4,0,%2\n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	"\n\
 2:"
 	: "=&r" (prev), "+m" (*p)
@@ -427,7 +427,7 @@
 	bne-	2f\n\
 	stdcx.	%4,0,%2\n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	"\n\
 2:"
 	: "=&r" (prev), "+m" (*p)
arch/powerpc/include/asm/synch.h +1 -1
Index: linux-2.6/arch/powerpc/include/asm/synch.h
===================================================================
@@ -38,7 +38,7 @@
 
 #ifdef CONFIG_SMP
 #define ISYNC_ON_SMP	"\n\tisync\n"
-#define LWSYNC_ON_SMP	stringify_in_c(LWSYNC) "\n"
+#define LWSYNC_ON_SMP	"\n\t" stringify_in_c(LWSYNC) "\n"
 #else
 #define ISYNC_ON_SMP
 #define LWSYNC_ON_SMP
arch/powerpc/include/asm/mutex.h +2 -2
Index: linux-2.6/arch/powerpc/include/asm/mutex.h
===================================================================
@@ -15,7 +15,7 @@
 	PPC405_ERR77(0,%1)
 "	stwcx.	%3,0,%1\n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	"\n\
 2:"
 	: "=&r" (t)
@@ -35,7 +35,7 @@
 	PPC405_ERR77(0,%1)
 "	stwcx.	%0,0,%1\n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (t)
 	: "r" (&v->counter)
 	: "cc", "memory");
arch/powerpc/mm/hash_low_64.S +3 -3
Index: linux-2.6/arch/powerpc/mm/hash_low_64.S
===================================================================
@@ -110,7 +110,7 @@
 	/* Write the linux PTE atomically (setting busy) */
 	stdcx.	r30,0,r6
 	bne-	1b
-	isync
+	lwsync
 
 	/* Step 2:
 	 *
@@ -393,7 +393,7 @@
 	/* Write the linux PTE atomically (setting busy) */
 	stdcx.	r30,0,r6
 	bne-	1b
-	isync
+	lwsync
 
 	/* Step 2:
 	 *
@@ -734,7 +734,7 @@
 	/* Write the linux PTE atomically (setting busy) */
 	stdcx.	r30,0,r6
 	bne-	1b
-	isync
+	lwsync
 
 	/* Step 2:
 	 *
#2 lore raw

Re: [patch 2/2] powerpc: replace isync with lwsync

Quoted 7 lines · Nick Piggin

OK, here is this patch again. You didn't think I'd let a 2% performance improvement be forgotten? :)

Anyway, patch won't work well on architecture without lwsync, but I won't bother fixing that kind of thing and making it merge worthy until you guys say something positive about it.

Quoted 6 lines

20 runs of tbench on the G5

unpatched AVG=920.37 STD=2.36
patched AVG=938.89 STD=3.33

(throughput in MB/s) This is a 1.9% throughput increase.

Definitely worth it believe. We could use a macro that uses michael new improvements on the CPU features code pathing so that the isync gets changed to lwsync on some CPUs based on the availability of it.

Cheers,
Ben.

Quoted 304 lines

---

Index: linux-2.6/arch/powerpc/include/asm/atomic.h
===================================================================

arch/powerpc/include/asm/atomic.h +12 -12
@@ -55,7 +55,7 @@
 	PPC405_ERR77(0,%2)
 "	stwcx.	%0,0,%2 \n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (t)
 	: "r" (a), "r" (&v->counter)
 	: "cc", "memory");
@@ -91,7 +91,7 @@
 	PPC405_ERR77(0,%2)
 "	stwcx.	%0,0,%2 \n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (t)
 	: "r" (a), "r" (&v->counter)
 	: "cc", "memory");
@@ -125,7 +125,7 @@
 	PPC405_ERR77(0,%1)
 "	stwcx.	%0,0,%1 \n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (t)
 	: "r" (&v->counter)
 	: "cc", "xer", "memory");
@@ -169,7 +169,7 @@
 	PPC405_ERR77(0,%1)
 "	stwcx.	%0,0,%1\n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (t)
 	: "r" (&v->counter)
 	: "cc", "xer", "memory");
@@ -202,7 +202,7 @@
 	PPC405_ERR77(0,%2)
 "	stwcx.	%0,0,%1 \n\
 	bne-	1b \n"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 "	subf	%0,%2,%0 \n\
 2:"
 	: "=&r" (t)
@@ -235,7 +235,7 @@
 	PPC405_ERR77(0,%1)
 "	stwcx.	%0,0,%1\n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	"\n\
 2:"	: "=&b" (t)
 	: "r" (&v->counter)
@@ -291,7 +291,7 @@
 	add	%0,%1,%0\n\
 	stdcx.	%0,0,%2 \n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (t)
 	: "r" (a), "r" (&v->counter)
 	: "cc", "memory");
@@ -325,7 +325,7 @@
 	subf	%0,%1,%0\n\
 	stdcx.	%0,0,%2 \n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (t)
 	: "r" (a), "r" (&v->counter)
 	: "cc", "memory");
@@ -357,7 +357,7 @@
 	addic	%0,%0,1\n\
 	stdcx.	%0,0,%1 \n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (t)
 	: "r" (&v->counter)
 	: "cc", "xer", "memory");
@@ -399,7 +399,7 @@
 	addic	%0,%0,-1\n\
 	stdcx.	%0,0,%1\n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (t)
 	: "r" (&v->counter)
 	: "cc", "xer", "memory");
@@ -425,7 +425,7 @@
 	blt-	2f\n\
 	stdcx.	%0,0,%1\n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	"\n\
 2:"	: "=&r" (t)
 	: "r" (&v->counter)
@@ -458,7 +458,7 @@
 	add	%0,%2,%0 \n"
 "	stdcx.	%0,0,%1 \n\
 	bne-	1b \n"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 "	subf	%0,%2,%0 \n\
 2:"
 	: "=&r" (t)

Index: linux-2.6/arch/powerpc/include/asm/bitops.h
===================================================================

arch/powerpc/include/asm/bitops.h +4 -4
@@ -139,7 +139,7 @@
 	PPC405_ERR77(0,%3)
 	PPC_STLCX "%1,0,%3 \n"
 	"bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (old), "=&r" (t)
 	: "r" (mask), "r" (p)
 	: "cc", "memory");
@@ -160,7 +160,7 @@
 	PPC405_ERR77(0,%3)
 	PPC_STLCX "%1,0,%3 \n"
 	"bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (old), "=&r" (t)
 	: "r" (mask), "r" (p)
 	: "cc", "memory");
@@ -182,7 +182,7 @@
 	PPC405_ERR77(0,%3)
 	PPC_STLCX "%1,0,%3 \n"
 	"bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (old), "=&r" (t)
 	: "r" (mask), "r" (p)
 	: "cc", "memory");
@@ -204,7 +204,7 @@
 	PPC405_ERR77(0,%3)
 	PPC_STLCX "%1,0,%3 \n"
 	"bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (old), "=&r" (t)
 	: "r" (mask), "r" (p)
 	: "cc", "memory");

Index: linux-2.6/arch/powerpc/include/asm/futex.h
===================================================================

arch/powerpc/include/asm/futex.h +1 -1
@@ -97,7 +97,7 @@
         PPC405_ERR77(0,%2)
 "2:     stwcx.  %4,0,%2\n\
         bne-    1b\n"
-        ISYNC_ON_SMP
+        LWSYNC_ON_SMP
 "3:	.section .fixup,\"ax\"\n\
 4:	li	%0,%5\n\
 	b	3b\n\

Index: linux-2.6/arch/powerpc/include/asm/spinlock.h
===================================================================

arch/powerpc/include/asm/spinlock.h +3 -3
@@ -65,7 +65,7 @@
 	bne-		2f\n\
 	stwcx.		%1,0,%2\n\
 	bne-		1b\n\
-	isync\n\
+	lwsync\n\
 2:"	: "=&r" (tmp)
 	: "r" (token), "r" (&lock->slock)
 	: "cr0", "memory");
@@ -193,7 +193,7 @@
 	PPC405_ERR77(0,%1)
 "	stwcx.		%0,0,%1\n\
 	bne-		1b\n\
-	isync\n\
+	lwsync\n\
 2:"	: "=&r" (tmp)
 	: "r" (&rw->lock)
 	: "cr0", "xer", "memory");
@@ -217,7 +217,7 @@
 	PPC405_ERR77(0,%1)
 "	stwcx.		%1,0,%2\n\
 	bne-		1b\n\
-	isync\n\
+	lwsync\n\
 2:"	: "=&r" (tmp)
 	: "r" (token), "r" (&rw->lock)
 	: "cr0", "memory");

Index: linux-2.6/arch/powerpc/include/asm/system.h
===================================================================

arch/powerpc/include/asm/system.h +4 -4
@@ -246,7 +246,7 @@
 	PPC405_ERR77(0,%2)
 "	stwcx.	%3,0,%2 \n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (prev), "+m" (*(volatile unsigned int *)p)
 	: "r" (p), "r" (val)
 	: "cc", "memory");
@@ -289,7 +289,7 @@
 	PPC405_ERR77(0,%2)
 "	stdcx.	%3,0,%2 \n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (prev), "+m" (*(volatile unsigned long *)p)
 	: "r" (p), "r" (val)
 	: "cc", "memory");
@@ -382,7 +382,7 @@
 	PPC405_ERR77(0,%2)
 "	stwcx.	%4,0,%2\n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	"\n\
 2:"
 	: "=&r" (prev), "+m" (*p)
@@ -427,7 +427,7 @@
 	bne-	2f\n\
 	stdcx.	%4,0,%2\n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	"\n\
 2:"
 	: "=&r" (prev), "+m" (*p)

Index: linux-2.6/arch/powerpc/include/asm/synch.h
===================================================================

arch/powerpc/include/asm/synch.h +1 -1
@@ -38,7 +38,7 @@
 
 #ifdef CONFIG_SMP
 #define ISYNC_ON_SMP	"\n\tisync\n"
-#define LWSYNC_ON_SMP	stringify_in_c(LWSYNC) "\n"
+#define LWSYNC_ON_SMP	"\n\t" stringify_in_c(LWSYNC) "\n"
 #else
 #define ISYNC_ON_SMP
 #define LWSYNC_ON_SMP

Index: linux-2.6/arch/powerpc/include/asm/mutex.h
===================================================================

arch/powerpc/include/asm/mutex.h +2 -2
@@ -15,7 +15,7 @@
 	PPC405_ERR77(0,%1)
 "	stwcx.	%3,0,%1\n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	"\n\
 2:"
 	: "=&r" (t)
@@ -35,7 +35,7 @@
 	PPC405_ERR77(0,%1)
 "	stwcx.	%0,0,%1\n\
 	bne-	1b"
-	ISYNC_ON_SMP
+	LWSYNC_ON_SMP
 	: "=&r" (t)
 	: "r" (&v->counter)
 	: "cc", "memory");

Index: linux-2.6/arch/powerpc/mm/hash_low_64.S
===================================================================

arch/powerpc/mm/hash_low_64.S +3 -3
@@ -110,7 +110,7 @@
 	/* Write the linux PTE atomically (setting busy) */
 	stdcx.	r30,0,r6
 	bne-	1b
-	isync
+	lwsync
 
 	/* Step 2:
 	 *
@@ -393,7 +393,7 @@
 	/* Write the linux PTE atomically (setting busy) */
 	stdcx.	r30,0,r6
 	bne-	1b
-	isync
+	lwsync
 
 	/* Step 2:
 	 *
@@ -734,7 +734,7 @@
 	/* Write the linux PTE atomically (setting busy) */
 	stdcx.	r30,0,r6
 	bne-	1b
-	isync
+	lwsync
 
 	/* Step 2:
 	 *
#3 lore raw

Re: [patch 2/2] powerpc: replace isync with lwsync

Quoted 10 lines · Benjamin Herrenschmidt
Quoted 9 lines · Nick Piggin

OK, here is this patch again. You didn't think I'd let a 2% performance improvement be forgotten? :)

Anyway, patch won't work well on architecture without lwsync, but I won't bother fixing that kind of thing and making it merge worthy until you guys say something positive about it.

20 runs of tbench on the G5

Quoted 8 lines
Quoted 4 lines

unpatched AVG=920.37 STD=2.36
patched AVG=938.89 STD=3.33

(throughput in MB/s) This is a 1.9% throughput increase.

Definitely worth it believe. We could use a macro that uses michael new improvements on the CPU features code pathing so that the isync gets changed to lwsync on some CPUs based on the availability of it.

OK. I guess the interesting part about this is that I can't find any IBM documentation for lwsync capable CPUs that suggest using this pattern for acquire locking. It would be interesting to know whether it helps other CPUs...

#4 lore raw
In reply to Nick Piggin

Allright, sorry for the delay, I had those stored into my "need more than half a brain cell for review" list and only got to them today :-)

Quoted 6 lines · Nick Piggin

Using lwsync, isync sequence in a microbenchmark is 5 times faster on my G5 than using sync for smp_mb. Although it takes more instructions.

Running tbench with 4 clients on my 4 core G5 (20 times) gives the following:

Quoted 6 lines

unpatched AVG=920.33 STD=2.36
patched AVG=921.27 STD=2.77

So not a big improvement here, actually it could even be in the noise. But other workloads or systems might see a bigger win, and the patch maybe is interesting or could be improved, so I'll ask for comments.

So not a huge objection here, however I have some doubts as to whether this will be worthwhile on power5,6,7 since those optimized somewhat the behaviour of the full sync. Since anything older than power4 doesn't have lwsync, that potentially makes it not worth the pain.

But I need to measure to be sure... it might be that newer embedded processors that support lwsync and SMP (and that are using a different pipeline structure) might benefit from this. I'll try to run some tests later this week or next week, but ping me in case I forget.

Now what would be worth doing is to also try using a twi;isync sequence like we do to order MMIO reads, see if it's any better than cmp/branch

Cheers,
Ben.

Quoted 22 lines

---
Index: linux-2.6/arch/powerpc/include/asm/system.h
===================================================================

arch/powerpc/include/asm/system.h +9 -0
@@ -52,7 +52,16 @@
 #    define SMPWMB      eieio
 #endif
 
+#ifdef __powerpc64__
+#define smp_mb()	__asm__ __volatile__ (				    \
+					"1:	lwsync			\n" \
+					"	cmpw	0,%%r0,%%r0	\n" \
+					"	bne-	1b		\n" \
+					"	isync			\n" \
+					: : : "memory")
+#else
 #define smp_mb()	mb()
+#endif
 #define smp_rmb()	__asm__ __volatile__ (stringify_in_c(LWSYNC) : : :"memory")
 #define smp_wmb()	__asm__ __volatile__ (stringify_in_c(SMPWMB) : : :"memory")
 #define smp_read_barrier_depends()	read_barrier_depends()
#5 lore raw
Quoted 2 lines · Benjamin Herrenschmidt

Allright, sorry for the delay, I had those stored into my "need more than half a brain cell for review" list and only got to them today :-)

No problem :)

Quoted 10 lines · Nick Piggin
Quoted 9 lines

Using lwsync, isync sequence in a microbenchmark is 5 times faster on my G5 than using sync for smp_mb. Although it takes more instructions.

Running tbench with 4 clients on my 4 core G5 (20 times) gives the following:

unpatched AVG=920.33 STD=2.36
patched AVG=921.27 STD=2.77

Quoted 8 lines
Quoted 3 lines

So not a big improvement here, actually it could even be in the noise. But other workloads or systems might see a bigger win, and the patch maybe is interesting or could be improved, so I'll ask for comments.

So not a huge objection here, however I have some doubts as to whether this will be worthwhile on power5,6,7 since those optimized somewhat the behaviour of the full sync. Since anything older than power4 doesn't have lwsync, that potentially makes it not worth the pain.

I would be interested to know. Avoiding sync when there *is* outstanding IO operations happening should be a win? (My test of tbench on localhost obviously wouldn't generate much MMIO).

I mean, even in the most optimised implementation possible, this sequence is less constraining than sync.

Quoted 4 lines

But I need to measure to be sure... it might be that newer embedded processors that support lwsync and SMP (and that are using a different pipeline structure) might benefit from this. I'll try to run some tests later this week or next week, but ping me in case I forget.

OK I'll ping you next week.

Quoted 2 lines

Now what would be worth doing is to also try using a twi;isync sequence like we do to order MMIO reads, see if it's any better than cmp/branch

Probably makes sense to use the same pattern.